From 7a0bb040575d00b0e3adb2cd474b3f4c84b3c838 Mon Sep 17 00:00:00 2001 From: ChenJie Date: Fri, 27 May 2022 20:19:30 +0800 Subject: [PATCH 1/3] mmap debug Signed-off-by: ChenJie --- linux-5.10/rk3568_patch/kernel.patch | 74 ++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/linux-5.10/rk3568_patch/kernel.patch b/linux-5.10/rk3568_patch/kernel.patch index a692478..7984c6a 100755 --- a/linux-5.10/rk3568_patch/kernel.patch +++ b/linux-5.10/rk3568_patch/kernel.patch @@ -1,3 +1,77 @@ +diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c +index d5ffaaab31a7d1ed433e8e7bf01ae1f52848cd22..45eb5e0c67e4d27b281ff9c6664ed145acda19e4 100644 +--- a/arch/arm64/kernel/sys.c ++++ b/arch/arm64/kernel/sys.c +@@ -22,10 +22,13 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, unsigned long, off) + { ++ unsigned long ret; + if (offset_in_page(off) != 0) + return -EINVAL; + +- return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); ++ ret = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); ++ pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, ret); ++ return ret; + } + + SYSCALL_DEFINE1(arm64_personality, unsigned int, personality) +diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c +index fc40386afb1bbacfaa69b10c2574295b67c0a74a..535df6d47b4ba377b4a2e6614a7825ddef382f91 100644 +--- a/arch/arm64/kernel/sys32.c ++++ b/arch/arm64/kernel/sys32.c +@@ -56,12 +56,16 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, unsigned long, off_4k) + { ++ unsigned long ret; + if (off_4k & (~PAGE_MASK >> 12)) + return -EINVAL; + + off_4k >>= (PAGE_SHIFT - 12); + +- return ksys_mmap_pgoff(addr, len, prot, flags, fd, off_4k); ++ ret = ksys_mmap_pgoff(addr, len, prot, flags, fd, off_4k); ++ pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, ret); ++ return ret; ++} + } + + #ifdef CONFIG_CPU_BIG_ENDIAN +diff --git a/mm/mmap.c b/mm/mmap.c +index e2c36409852bbae7b8eb48047e7d954268886332..ed1d06ed90e2ef1792bd19ae3dfe42b25987f8f7 100644 +--- a/mm/mmap.c ++++ b/mm/mmap.c +@@ -1592,7 +1592,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, + ((vm_flags & VM_LOCKED) || + (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) + *populate = len; +- pr_err("pid=%d, addr=%lx\n", current->pid, addr); ++ pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, addr); + return addr; + } + +@@ -1640,6 +1640,7 @@ unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, + flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE); + + retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); ++ pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, retval); + out_fput: + if (file) + fput(file); +diff --git a/mm/util.c b/mm/util.c +index 90792e4eaa252f9daa66632ba66d874c731a12c6..afddc41cd74e4cf138f2c304f454fa5c8013392c 100644 +--- a/mm/util.c ++++ b/mm/util.c +@@ -511,6 +511,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, + if (populate) + mm_populate(ret, populate); + } ++ pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, ret); + return ret; + } + diff --git a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt index 148191b0f..eee2a7f7c 100644 --- a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt -- Gitee From 14dfec2a32b3df27289b1ae2a389bddb2e091d48 Mon Sep 17 00:00:00 2001 From: ChenJie Date: Fri, 27 May 2022 20:40:51 +0800 Subject: [PATCH 2/3] mmap debug Signed-off-by: ChenJie --- linux-5.10/rk3568_patch/kernel.patch | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/linux-5.10/rk3568_patch/kernel.patch b/linux-5.10/rk3568_patch/kernel.patch index 7984c6a..28c52e1 100755 --- a/linux-5.10/rk3568_patch/kernel.patch +++ b/linux-5.10/rk3568_patch/kernel.patch @@ -70,8 +70,7 @@ index 90792e4eaa252f9daa66632ba66d874c731a12c6..afddc41cd74e4cf138f2c304f454fa5c } + pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, ret); return ret; - } - + } diff --git a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt index 148191b0f..eee2a7f7c 100644 --- a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt -- Gitee From afe7405a5a4636749401ab6b02945ea6befd9d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=9D=B0?= Date: Fri, 27 May 2022 12:44:44 +0000 Subject: [PATCH 3/3] update linux-5.10/rk3568_patch/kernel.patch. Signed-off-by: ChenJie --- linux-5.10/rk3568_patch/kernel.patch | 1351427 +---------------------- 1 file changed, 2 insertions(+), 1351425 deletions(-) diff --git a/linux-5.10/rk3568_patch/kernel.patch b/linux-5.10/rk3568_patch/kernel.patch index 28c52e1..6e050ba 100755 --- a/linux-5.10/rk3568_patch/kernel.patch +++ b/linux-5.10/rk3568_patch/kernel.patch @@ -70,7 +70,7 @@ index 90792e4eaa252f9daa66632ba66d874c731a12c6..afddc41cd74e4cf138f2c304f454fa5c } + pr_err("%s, pid=%d, addr=%lx\n", __FUNC__ ,current->pid, ret); return ret; - } + } diff --git a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt b/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt index 148191b0f..eee2a7f7c 100644 --- a/Documentation/devicetree/bindings/devfreq/event/rockchip-dfi.txt @@ -356480,1351427 +356480,4 @@ index 000000000..ffffee930 + MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev); +} + -+/* Inserting the waiter object into the given timeline */ -+static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new) -+{ -+ struct mali_timeline_waiter *waiter_prev; -+ struct mali_timeline_waiter *waiter_next; -+ -+ /* Waiter time must be between timeline head and tail, and there must -+ * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */ -+ MALI_DEBUG_ASSERT((waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN); -+ MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN); -+ -+ /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/ -+ waiter_prev = timeline->waiter_head; /* Insert new after waiter_prev */ -+ waiter_next = NULL; /* Insert new before waiter_next */ -+ -+ /* Iterating backwards from head (newest) to tail (oldest) until we -+ * find the correct spot to insert the new waiter */ -+ while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) { -+ waiter_next = waiter_prev; -+ waiter_prev = waiter_prev->timeline_prev; -+ } -+ -+ if (NULL == waiter_prev && NULL == waiter_next) { -+ /* list is empty */ -+ timeline->waiter_head = waiter_new; -+ timeline->waiter_tail = waiter_new; -+ } else if (NULL == waiter_next) { -+ /* insert at head */ -+ waiter_new->timeline_prev = timeline->waiter_head; -+ timeline->waiter_head->timeline_next = waiter_new; -+ timeline->waiter_head = waiter_new; -+ } else if (NULL == waiter_prev) { -+ /* insert at tail */ -+ waiter_new->timeline_next = timeline->waiter_tail; -+ timeline->waiter_tail->timeline_prev = waiter_new; -+ timeline->waiter_tail = waiter_new; -+ } else { -+ /* insert between */ -+ waiter_new->timeline_next = waiter_next; -+ waiter_new->timeline_prev = waiter_prev; -+ waiter_next->timeline_prev = waiter_new; -+ waiter_prev->timeline_next = waiter_new; -+ } -+} -+ -+static void mali_timeline_update_delayed_work(struct mali_timeline *timeline) -+{ -+ struct mali_timeline_system *system; -+ struct mali_timeline_tracker *oldest_tracker; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); -+ -+ system = timeline->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ /* Timer is disabled, early out. */ -+ if (!system->timer_enabled) return; -+ -+ oldest_tracker = timeline->tracker_tail; -+ if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) { -+ if (MALI_FALSE == oldest_tracker->timer_active) { -+ if (MALI_TRUE == timeline->timer_active) { -+ _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); -+ } -+ _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ); -+ oldest_tracker->timer_active = MALI_TRUE; -+ timeline->timer_active = MALI_TRUE; -+ } -+ } else if (MALI_TRUE == timeline->timer_active) { -+ _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work); -+ timeline->timer_active = MALI_FALSE; -+ } -+} -+ -+static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline) -+{ -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ MALI_DEBUG_CODE({ -+ struct mali_timeline_system *system = timeline->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ }); -+ -+ if (NULL != timeline->tracker_tail) { -+ /* Set oldest point to oldest tracker's point */ -+ timeline->point_oldest = timeline->tracker_tail->point; -+ } else { -+ /* No trackers, mark point list as empty */ -+ timeline->point_oldest = timeline->point_next; -+ } -+ -+ /* Release all waiters no longer on the timeline's point list. -+ * Releasing a waiter can trigger this function to be called again, so -+ * we do not store any pointers on stack. */ -+ while (NULL != timeline->waiter_tail) { -+ u32 waiter_time_relative; -+ u32 time_head_relative; -+ struct mali_timeline_waiter *waiter = timeline->waiter_tail; -+ -+ time_head_relative = timeline->point_next - timeline->point_oldest; -+ waiter_time_relative = waiter->point - timeline->point_oldest; -+ -+ if (waiter_time_relative < time_head_relative) { -+ /* This and all following waiters are on the point list, so we are done. */ -+ break; -+ } -+ -+ /* Remove waiter from timeline's waiter list. */ -+ if (NULL != waiter->timeline_next) { -+ waiter->timeline_next->timeline_prev = NULL; -+ } else { -+ /* This was the last waiter */ -+ timeline->waiter_head = NULL; -+ } -+ timeline->waiter_tail = waiter->timeline_next; -+ -+ /* Release waiter. This could activate a tracker, if this was -+ * the last waiter for the tracker. */ -+ schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); -+ } -+ -+ return schedule_mask; -+} -+ -+static mali_scheduler_mask mali_timeline_release_with_depended_point(struct mali_timeline_tracker *tracker) -+{ -+ struct mali_timeline *timeline; -+ struct mali_timeline_waiter *waiter; -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ -+ timeline = tracker->timeline; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id); -+ -+ MALI_DEBUG_CODE({ -+ struct mali_timeline_system *system = timeline->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ }); -+ -+ /* Only release the waiter that wait for the tracker. */ -+ waiter = timeline->waiter_tail; -+ while (NULL != waiter) { -+ if (waiter->point == tracker->point) { -+ -+ struct mali_timeline_waiter *waiter_next; -+ struct mali_timeline_waiter *waiter_prev; -+ -+ waiter_next = waiter->timeline_next; -+ waiter_prev = waiter->timeline_prev; -+ waiter->timeline_next = NULL; -+ waiter->timeline_prev = NULL; -+ -+ if (NULL != waiter_prev) { -+ waiter_prev->timeline_next = waiter_next; -+ } -+ -+ if (NULL != waiter_next) { -+ waiter_next->timeline_prev = waiter_prev; -+ } -+ -+ if (waiter == timeline->waiter_tail) -+ timeline->waiter_tail = waiter_next; -+ -+ if (waiter == timeline->waiter_head) -+ timeline->waiter_head = NULL; -+ -+ schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter); -+ waiter = waiter_next; -+ }else { -+ -+ waiter = waiter->timeline_next; -+ } -+ } -+ -+ return schedule_mask; -+} -+ -+void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, -+ mali_timeline_tracker_type type, -+ struct mali_timeline_fence *fence, -+ void *job) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ MALI_DEBUG_ASSERT_POINTER(job); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type); -+ -+ /* Zero out all tracker members. */ -+ _mali_osk_memset(tracker, 0, sizeof(*tracker)); -+ -+ tracker->type = type; -+ tracker->job = job; -+ tracker->trigger_ref_count = 1; /* Prevents any callback from trigging while adding it */ -+ tracker->os_tick_create = _mali_osk_time_tickcount(); -+ MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC); -+ -+ tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE; -+ -+ /* Copy fence. */ -+ if (NULL != fence) { -+ _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence)); -+ } -+} -+ -+mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker) -+{ -+ struct mali_timeline *timeline; -+ struct mali_timeline_system *system; -+ struct mali_timeline_tracker *tracker_next, *tracker_prev; -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ u32 tid = _mali_osk_get_tid(); -+ -+ /* Upon entry a group lock will be held, but not a scheduler lock. */ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); -+ -+ /* Tracker should have been triggered */ -+ MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count); -+ -+ /* All waiters should have been released at this point */ -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); -+ -+ MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job)); -+ -+ timeline = tracker->timeline; -+ if (NULL == timeline) { -+ /* Tracker was not on a timeline, there is nothing to release. */ -+ return MALI_SCHEDULER_MASK_EMPTY; -+ } -+ -+ system = timeline->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ /* Tracker should still be on timeline */ -+ MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); -+ MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, tracker->point)); -+ -+ /* Tracker is no longer valid. */ -+ MALI_DEBUG_CODE(tracker->magic = 0); -+ -+ tracker_next = tracker->timeline_next; -+ tracker_prev = tracker->timeline_prev; -+ tracker->timeline_next = NULL; -+ tracker->timeline_prev = NULL; -+ -+ /* Removing tracker from timeline's tracker list */ -+ if (NULL == tracker_next) { -+ /* This tracker was the head */ -+ timeline->tracker_head = tracker_prev; -+ } else { -+ tracker_next->timeline_prev = tracker_prev; -+ } -+ -+ if (NULL == tracker_prev) { -+ /* This tracker was the tail */ -+ timeline->tracker_tail = tracker_next; -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ /* Update the timeline's oldest time and release any waiters */ -+ schedule_mask |= mali_timeline_update_oldest_point(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ } else { -+ tracker_prev->timeline_next = tracker_next; -+ if (MALI_TIMELINE_SOFT == tracker->timeline->id) { -+ /* Use the signaled soft tracker to release the depended soft waiter */ -+ schedule_mask |= mali_timeline_release_with_depended_point(tracker); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ } -+ } -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ /* Update delayed work only when it is the soft job timeline */ -+ if (MALI_TIMELINE_SOFT == tracker->timeline->id) { -+ mali_timeline_update_delayed_work(tracker->timeline); -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ return schedule_mask; -+} -+ -+void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system, -+ struct mali_timeline_waiter *tail, -+ struct mali_timeline_waiter *head) -+{ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(head); -+ MALI_DEBUG_ASSERT_POINTER(tail); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ head->tracker_next = system->waiter_empty_list; -+ system->waiter_empty_list = tail; -+} -+ -+static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker) -+{ -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ struct mali_timeline_system *system; -+ struct mali_timeline *timeline; -+ u32 tid = _mali_osk_get_tid(); -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); -+ -+ system = tracker->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ tracker->os_tick_activate = _mali_osk_time_tickcount(); -+ -+ if (NULL != tracker->waiter_head) { -+ mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head); -+ tracker->waiter_head = NULL; -+ tracker->waiter_tail = NULL; -+ } -+ -+ switch (tracker->type) { -+ case MALI_TIMELINE_TRACKER_GP: -+ schedule_mask = mali_scheduler_activate_gp_job((struct mali_gp_job *) tracker->job); -+ -+ _mali_osk_atomic_dec(&gp_tracker_count); -+ break; -+ case MALI_TIMELINE_TRACKER_PP: -+ if (mali_pp_job_is_virtual((struct mali_pp_job *)tracker->job)) { -+ _mali_osk_atomic_dec(&virt_pp_tracker_count); -+ } else { -+ _mali_osk_atomic_dec(&phy_pp_tracker_count); -+ } -+ schedule_mask = mali_scheduler_activate_pp_job((struct mali_pp_job *) tracker->job); -+ break; -+ case MALI_TIMELINE_TRACKER_SOFT: -+ timeline = tracker->timeline; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ schedule_mask |= mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job); -+ -+ /* Start a soft timer to make sure the soft job be released in a limited time */ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ mali_timeline_update_delayed_work(timeline); -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ break; -+ case MALI_TIMELINE_TRACKER_WAIT: -+ mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job); -+ break; -+ case MALI_TIMELINE_TRACKER_SYNC: -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job); -+#else -+ MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type)); -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ break; -+ default: -+ MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type)); -+ break; -+ } -+ -+ return schedule_mask; -+} -+ -+void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker) -+{ -+ u32 tid = _mali_osk_get_tid(); -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); -+ tracker->trigger_ref_count++; -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+} -+ -+mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error) -+{ -+ u32 tid = _mali_osk_get_tid(); -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); -+ tracker->trigger_ref_count--; -+ -+ tracker->activation_error |= activation_error; -+ -+ if (0 == tracker->trigger_ref_count) { -+ schedule_mask |= mali_timeline_tracker_activate(tracker); -+ tracker = NULL; -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ return schedule_mask; -+} -+ -+void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence) -+{ -+ u32 i; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ MALI_DEBUG_ASSERT_POINTER(uk_fence); -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ fence->points[i] = uk_fence->points[i]; -+ } -+ -+ fence->sync_fd = uk_fence->sync_fd; -+} -+ -+struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session) -+{ -+ u32 i; -+ struct mali_timeline_system *system; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n")); -+ -+ system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system)); -+ if (NULL == system) { -+ return NULL; -+ } -+ -+ system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM); -+ if (NULL == system->spinlock) { -+ mali_timeline_system_destroy(system); -+ return NULL; -+ } -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i); -+ if (NULL == system->timelines[i]) { -+ mali_timeline_system_destroy(system); -+ return NULL; -+ } -+ } -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ system->signaled_sync_tl = mali_sync_timeline_create(NULL, "mali-always-signaled"); -+ if (NULL == system->signaled_sync_tl) { -+ mali_timeline_system_destroy(system); -+ return NULL; -+ } -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ system->waiter_empty_list = NULL; -+ system->session = session; -+ system->timer_enabled = MALI_TRUE; -+ -+ system->wait_queue = _mali_osk_wait_queue_init(); -+ if (NULL == system->wait_queue) { -+ mali_timeline_system_destroy(system); -+ return NULL; -+ } -+ -+ return system; -+} -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE) -+/** -+ * Check if there are any trackers left on timeline. -+ * -+ * Used as a wait queue conditional. -+ * -+ * @param data Timeline. -+ * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not. -+ */ -+static mali_bool mali_timeline_has_no_trackers(void *data) -+{ -+ struct mali_timeline *timeline = (struct mali_timeline *) data; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ return mali_timeline_is_empty(timeline); -+} -+#if defined(CONFIG_SYNC) ||defined(CONFIG_SYNC_FILE) -+/** -+ * Cancel sync fence waiters waited upon by trackers on all timelines. -+ * -+ * Will return after all timelines have no trackers left. -+ * -+ * @param system Timeline system. -+ */ -+static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system) -+{ -+ u32 i; -+ u32 tid = _mali_osk_get_tid(); -+ struct mali_timeline_tracker *tracker, *tracker_next; -+ _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list); -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ MALI_DEBUG_ASSERT(system->session->is_aborting); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ /* Cancel sync fence waiters. */ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ struct mali_timeline *timeline = system->timelines[i]; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ tracker_next = timeline->tracker_tail; -+ while (NULL != tracker_next) { -+ tracker = tracker_next; -+ tracker_next = tracker->timeline_next; -+ -+ if (NULL == tracker->sync_fence) continue; -+ -+ MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker)); -+ -+ /* Cancel sync fence waiter. */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { -+#else -+ if (0 == mali_internal_sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) { -+#endif -+ /* Callback was not called, move tracker to local list. */ -+ _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list); -+ } -+ } -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */ -+ _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) { -+ mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter); -+ } -+ -+ /* Sleep until all sync fence callbacks are done and all timelines are empty. */ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ struct mali_timeline *timeline = system->timelines[i]; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); -+ } -+} -+ -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+static void mali_timeline_cancel_dma_fence_waiters(struct mali_timeline_system *system) -+{ -+ u32 i, j; -+ u32 tid = _mali_osk_get_tid(); -+ struct mali_pp_job *pp_job = NULL; -+ struct mali_pp_job *next_pp_job = NULL; -+ struct mali_timeline *timeline = NULL; -+ struct mali_timeline_tracker *tracker, *tracker_next; -+ _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_job_list); -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ MALI_DEBUG_ASSERT(system->session->is_aborting); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ /* Cancel dma fence waiters. */ -+ timeline = system->timelines[MALI_TIMELINE_PP]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ tracker_next = timeline->tracker_tail; -+ while (NULL != tracker_next) { -+ mali_bool fence_is_signaled = MALI_TRUE; -+ tracker = tracker_next; -+ tracker_next = tracker->timeline_next; -+ -+ if (NULL == tracker->waiter_dma_fence) continue; -+ pp_job = (struct mali_pp_job *)tracker->job; -+ MALI_DEBUG_ASSERT_POINTER(pp_job); -+ MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling dma fence waiter for tracker 0x%08X.\n", tracker)); -+ -+ for (j = 0; j < pp_job->dma_fence_context.num_dma_fence_waiter; j++) { -+ if (pp_job->dma_fence_context.mali_dma_fence_waiters[j]) { -+ /* Cancel a previously callback from the fence. -+ * This function returns true if the callback is successfully removed, -+ * or false if the fence has already been signaled. -+ */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ bool ret = dma_fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence, -+ &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base); -+ -+#else -+ bool ret = fence_remove_callback(pp_job->dma_fence_context.mali_dma_fence_waiters[j]->fence, -+ &pp_job->dma_fence_context.mali_dma_fence_waiters[j]->base); -+#endif -+ if (ret) { -+ fence_is_signaled = MALI_FALSE; -+ } -+ } -+ } -+ -+ /* Callbacks were not called, move pp job to local list. */ -+ if (MALI_FALSE == fence_is_signaled) -+ _mali_osk_list_add(&pp_job->list, &pp_job_list); -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ /* Manually call dma fence callback in order to release waiter and trigger activation of tracker. */ -+ _MALI_OSK_LIST_FOREACHENTRY(pp_job, next_pp_job, &pp_job_list, struct mali_pp_job, list) { -+ mali_timeline_dma_fence_callback((void *)pp_job); -+ } -+ -+ /* Sleep until all dma fence callbacks are done and all timelines are empty. */ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ struct mali_timeline *timeline = system->timelines[i]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline); -+ } -+} -+#endif -+#endif -+void mali_timeline_system_abort(struct mali_timeline_system *system) -+{ -+ MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid();); -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ MALI_DEBUG_ASSERT(system->session->is_aborting); -+ -+ MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session)); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ mali_timeline_cancel_sync_fence_waiters(system); -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+ mali_timeline_cancel_dma_fence_waiters(system); -+#endif -+ -+ /* Should not be any waiters or trackers left at this point. */ -+ MALI_DEBUG_CODE({ -+ u32 i; -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) -+ { -+ struct mali_timeline *timeline = system->timelines[i]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next); -+ MALI_DEBUG_ASSERT(NULL == timeline->tracker_head); -+ MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail); -+ MALI_DEBUG_ASSERT(NULL == timeline->waiter_head); -+ MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail); -+ } -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ }); -+} -+ -+void mali_timeline_system_destroy(struct mali_timeline_system *system) -+{ -+ u32 i; -+ struct mali_timeline_waiter *waiter, *next; -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ u32 tid = _mali_osk_get_tid(); -+#endif -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ -+ MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n")); -+ -+ if (NULL != system) { -+ -+ /* There should be no waiters left on this queue. */ -+ if (NULL != system->wait_queue) { -+ _mali_osk_wait_queue_term(system->wait_queue); -+ system->wait_queue = NULL; -+ } -+ -+ /* Free all waiters in empty list */ -+ waiter = system->waiter_empty_list; -+ while (NULL != waiter) { -+ next = waiter->tracker_next; -+ _mali_osk_free(waiter); -+ waiter = next; -+ } -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (NULL != system->signaled_sync_tl) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_timeline_destroy(system->signaled_sync_tl); -+#else -+ mali_internal_sync_timeline_destroy(system->signaled_sync_tl); -+#endif -+ } -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ if ((NULL != system->timelines[i]) && (NULL != system->timelines[i]->spinlock)) { -+ mali_spinlock_reentrant_wait(system->timelines[i]->spinlock, tid); -+ system->timelines[i]->destroyed = MALI_TRUE; -+ mali_spinlock_reentrant_signal(system->timelines[i]->spinlock, tid); -+ } -+ } -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ if (NULL != system->timelines[i]) { -+ mali_timeline_destroy(system->timelines[i]); -+ } -+ } -+ -+ if (NULL != system->spinlock) { -+ mali_spinlock_reentrant_term(system->spinlock); -+ } -+ -+ _mali_osk_free(system); -+ } -+} -+ -+/** -+ * Find how many waiters are needed for a given fence. -+ * -+ * @param fence The fence to check. -+ * @return Number of waiters needed for fence. -+ */ -+static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence) -+{ -+ u32 i, num_waiters = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ if (MALI_TIMELINE_NO_POINT != fence->points[i]) { -+ ++num_waiters; -+ } -+ } -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (-1 != fence->sync_fd) ++num_waiters; -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ return num_waiters; -+} -+ -+static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system) -+{ -+ struct mali_timeline_waiter *waiter; -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ waiter = system->waiter_empty_list; -+ if (NULL != waiter) { -+ /* Remove waiter from empty list and zero it */ -+ system->waiter_empty_list = waiter->tracker_next; -+ _mali_osk_memset(waiter, 0, sizeof(*waiter)); -+ } -+ -+ /* Return NULL if list was empty. */ -+ return waiter; -+} -+ -+static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system, -+ struct mali_timeline_waiter **tail, -+ struct mali_timeline_waiter **head, -+ int max_num_waiters) -+{ -+ u32 i, tid = _mali_osk_get_tid(); -+ mali_bool do_alloc; -+ struct mali_timeline_waiter *waiter; -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(tail); -+ MALI_DEBUG_ASSERT_POINTER(head); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ *head = *tail = NULL; -+ do_alloc = MALI_FALSE; -+ i = 0; -+ while (i < max_num_waiters) { -+ if (MALI_FALSE == do_alloc) { -+ waiter = mali_timeline_system_get_zeroed_waiter(system); -+ if (NULL == waiter) { -+ do_alloc = MALI_TRUE; -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ continue; -+ } -+ } else { -+ waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter)); -+ if (NULL == waiter) break; -+ } -+ ++i; -+ if (NULL == *tail) { -+ *tail = waiter; -+ *head = waiter; -+ } else { -+ (*head)->tracker_next = waiter; -+ *head = waiter; -+ } -+ } -+ if (MALI_TRUE == do_alloc) { -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ } -+} -+ -+/** -+ * Create waiters for the given tracker. The tracker is activated when all waiters are release. -+ * -+ * @note Tracker can potentially be activated before this function returns. -+ * -+ * @param system Timeline system. -+ * @param tracker Tracker we will create waiters for. -+ * @param waiter_tail List of pre-allocated waiters. -+ * @param waiter_head List of pre-allocated waiters. -+ */ -+static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system, -+ struct mali_timeline_tracker *tracker, -+ struct mali_timeline_waiter *waiter_tail, -+ struct mali_timeline_waiter *waiter_head) -+{ -+ int i; -+ u32 tid = _mali_osk_get_tid(); -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence = NULL; -+#else -+ struct mali_internal_sync_fence *sync_fence = NULL; -+#endif -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_head); -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); -+ MALI_DEBUG_ASSERT(NULL != tracker->job); -+ -+ /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter -+ * into the timelines sorted list of waiters */ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ mali_timeline_point point; -+ struct mali_timeline *timeline; -+ struct mali_timeline_waiter *waiter; -+ -+ /* Get point on current timeline from tracker's fence. */ -+ point = tracker->fence.points[i]; -+ -+ if (likely(MALI_TIMELINE_NO_POINT == point)) { -+ /* Fence contains no point on this timeline so we don't need a waiter. */ -+ continue; -+ } -+ -+ timeline = system->timelines[i]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { -+ MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", -+ point, timeline->point_oldest, timeline->point_next)); -+ continue; -+ } -+ -+ if (likely(mali_timeline_is_point_released(timeline, point))) { -+ /* Tracker representing the point has been released so we don't need a -+ * waiter. */ -+ continue; -+ } -+ -+ if ((MALI_TIMELINE_SOFT == timeline->id) && mali_timeline_is_tracker_released(timeline, point)) { -+ /* The tracker that the point related to has already been released, so no need to a waiter. */ -+ continue; -+ } -+ -+ /* The point is on timeline. */ -+ MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point)); -+ -+ /* Get a new zeroed waiter object. */ -+ if (likely(NULL != waiter_tail)) { -+ waiter = waiter_tail; -+ waiter_tail = waiter_tail->tracker_next; -+ } else { -+ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); -+ continue; -+ } -+ -+ /* Yanking the trigger ref count of the tracker. */ -+ tracker->trigger_ref_count++; -+ -+ waiter->point = point; -+ waiter->tracker = tracker; -+ -+ /* Insert waiter on tracker's singly-linked waiter list. */ -+ if (NULL == tracker->waiter_head) { -+ /* list is empty */ -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); -+ tracker->waiter_tail = waiter; -+ } else { -+ tracker->waiter_head->tracker_next = waiter; -+ } -+ tracker->waiter_head = waiter; -+ -+ /* Add waiter to timeline. */ -+ mali_timeline_insert_waiter(timeline, waiter); -+ } -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (-1 != tracker->fence.sync_fd) { -+ int ret; -+ struct mali_timeline_waiter *waiter; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence = sync_fence_fdget(tracker->fence.sync_fd); -+#else -+ sync_fence = mali_internal_sync_fence_fdget(tracker->fence.sync_fd); -+#endif -+ if (unlikely(NULL == sync_fence)) { -+ MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd)); -+ goto exit; -+ } -+ -+ /* Check if we have a zeroed waiter object available. */ -+ if (unlikely(NULL == waiter_tail)) { -+ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); -+ goto exit; -+ } -+ -+ /* Start asynchronous wait that will release waiter when the fence is signaled. */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); -+ ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); -+#else -+ mali_internal_sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback); -+ ret = mali_internal_sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter); -+#endif -+ if (1 == ret) { -+ /* Fence already signaled, no waiter needed. */ -+ tracker->fence.sync_fd = -1; -+ goto exit; -+ } else if (0 != ret) { -+ MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret)); -+ tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; -+ goto exit; -+ } -+ -+ /* Grab new zeroed waiter object. */ -+ waiter = waiter_tail; -+ waiter_tail = waiter_tail->tracker_next; -+ -+ /* Increase the trigger ref count of the tracker. */ -+ tracker->trigger_ref_count++; -+ -+ waiter->point = MALI_TIMELINE_NO_POINT; -+ waiter->tracker = tracker; -+ -+ /* Insert waiter on tracker's singly-linked waiter list. */ -+ if (NULL == tracker->waiter_head) { -+ /* list is empty */ -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); -+ tracker->waiter_tail = waiter; -+ } else { -+ tracker->waiter_head->tracker_next = waiter; -+ } -+ tracker->waiter_head = waiter; -+ -+ /* Also store waiter in separate field for easy access by sync callback. */ -+ tracker->waiter_sync = waiter; -+ -+ /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */ -+ tracker->sync_fence = sync_fence; -+ -+ sync_fence = NULL; -+ } -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE)*/ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+ if ((NULL != tracker->timeline) && (MALI_TIMELINE_PP == tracker->timeline->id)) { -+ -+ struct mali_pp_job *job = (struct mali_pp_job *)tracker->job; -+ -+ if (0 < job->dma_fence_context.num_dma_fence_waiter) { -+ struct mali_timeline_waiter *waiter; -+ /* Check if we have a zeroed waiter object available. */ -+ if (unlikely(NULL == waiter_tail)) { -+ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n")); -+ goto exit; -+ } -+ -+ /* Grab new zeroed waiter object. */ -+ waiter = waiter_tail; -+ waiter_tail = waiter_tail->tracker_next; -+ -+ /* Increase the trigger ref count of the tracker. */ -+ tracker->trigger_ref_count++; -+ -+ waiter->point = MALI_TIMELINE_NO_POINT; -+ waiter->tracker = tracker; -+ -+ /* Insert waiter on tracker's singly-linked waiter list. */ -+ if (NULL == tracker->waiter_head) { -+ /* list is empty */ -+ MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail); -+ tracker->waiter_tail = waiter; -+ } else { -+ tracker->waiter_head->tracker_next = waiter; -+ } -+ tracker->waiter_head = waiter; -+ -+ /* Also store waiter in separate field for easy access by sync callback. */ -+ tracker->waiter_dma_fence = waiter; -+ } -+ } -+#endif /* defined(CONFIG_MALI_DMA_BUF_FENCE)*/ -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) ||defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+exit: -+#endif /* defined(CONFIG_MALI_DMA_BUF_FENCE) || defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ if (NULL != waiter_tail) { -+ mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head); -+ } -+ -+ /* Release the initial trigger ref count. */ -+ tracker->trigger_ref_count--; -+ -+ /* If there were no waiters added to this tracker we activate immediately. */ -+ if (0 == tracker->trigger_ref_count) { -+ schedule_mask |= mali_timeline_tracker_activate(tracker); -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (NULL != sync_fence) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence_put(sync_fence); -+#else -+ fput(sync_fence->file); -+#endif -+ } -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE); -+} -+ -+mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, -+ struct mali_timeline_tracker *tracker, -+ enum mali_timeline_id timeline_id) -+{ -+ int num_waiters = 0; -+ struct mali_timeline_waiter *waiter_tail, *waiter_head; -+ u32 tid = _mali_osk_get_tid(); -+ -+ mali_timeline_point point = MALI_TIMELINE_NO_POINT; -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic); -+ -+ MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id)); -+ -+ MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count); -+ tracker->system = system; -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ num_waiters = mali_timeline_fence_num_waiters(&tracker->fence); -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+ if (MALI_TIMELINE_PP == timeline_id) { -+ struct mali_pp_job *job = (struct mali_pp_job *)tracker->job; -+ if (0 < job->dma_fence_context.num_dma_fence_waiter) -+ num_waiters++; -+ } -+#endif -+ -+ /* Allocate waiters. */ -+ mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ /* Add tracker to timeline. This will allocate a point for the tracker on the timeline. If -+ * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the -+ * point will be MALI_TIMELINE_NO_POINT. -+ * -+ * NOTE: the tracker can fail to be added if the timeline is full. If this happens, the -+ * point will be MALI_TIMELINE_NO_POINT. */ -+ MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE); -+ if (likely(timeline_id < MALI_TIMELINE_MAX)) { -+ struct mali_timeline *timeline = system->timelines[timeline_id]; -+ mali_timeline_insert_tracker(timeline, tracker); -+ MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline)); -+ } -+ -+ point = tracker->point; -+ -+ /* Create waiters for tracker based on supplied fence. Each waiter will increase the -+ * trigger ref count. */ -+ mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head); -+ tracker = NULL; -+ -+ /* At this point the tracker object might have been freed so we should no longer -+ * access it. */ -+ -+ -+ /* The tracker will always be activated after calling add_tracker, even if NO_POINT is -+ * returned. */ -+ return point; -+} -+ -+static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system, -+ struct mali_timeline_waiter *waiter) -+{ -+ struct mali_timeline_tracker *tracker; -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(waiter); -+ -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system)); -+ -+ tracker = waiter->tracker; -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ /* At this point the waiter has been removed from the timeline's waiter list, but it is -+ * still on the tracker's waiter list. All of the tracker's waiters will be released when -+ * the tracker is activated. */ -+ -+ waiter->point = MALI_TIMELINE_NO_POINT; -+ waiter->tracker = NULL; -+ -+ tracker->trigger_ref_count--; -+ if (0 == tracker->trigger_ref_count) { -+ /* This was the last waiter; activate tracker */ -+ schedule_mask |= mali_timeline_tracker_activate(tracker); -+ tracker = NULL; -+ } -+ -+ return schedule_mask; -+} -+ -+mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, -+ enum mali_timeline_id timeline_id) -+{ -+ mali_timeline_point point; -+ struct mali_timeline *timeline; -+ u32 tid = _mali_osk_get_tid(); -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ if (MALI_TIMELINE_MAX <= timeline_id) { -+ return MALI_TIMELINE_NO_POINT; -+ } -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ timeline = system->timelines[timeline_id]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ point = MALI_TIMELINE_NO_POINT; -+ if (timeline->point_oldest != timeline->point_next) { -+ point = timeline->point_next - 1; -+ if (MALI_TIMELINE_NO_POINT == point) point--; -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ return point; -+} -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+static void mali_timeline_do_sync_fence_callback(void *arg) -+{ -+ _MALI_OSK_LIST_HEAD_STATIC_INIT(list); -+ struct mali_timeline_tracker *tracker; -+ struct mali_timeline_tracker *tmp_tracker; -+ u32 tid = _mali_osk_get_tid(); -+ -+ MALI_IGNORE(arg); -+ -+ /* -+ * Quickly "unhook" the jobs pending to be deleted, so we can release -+ * the lock before we start deleting the job objects -+ * (without any locks held) -+ */ -+ _mali_osk_spinlock_irq_lock(sync_fence_callback_list_lock); -+ _mali_osk_list_move_list(&sync_fence_callback_queue, &list); -+ _mali_osk_spinlock_irq_unlock(sync_fence_callback_list_lock); -+ -+ _MALI_OSK_LIST_FOREACHENTRY(tracker, tmp_tracker, &list, -+ struct mali_timeline_tracker, sync_fence_signal_list) { -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ mali_bool is_aborting = MALI_FALSE; -+ int fence_status = 0; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence = NULL; -+#else -+ struct mali_internal_sync_fence *sync_fence = NULL; -+#endif -+ struct mali_timeline_system *system = NULL; -+ struct mali_timeline_waiter *waiter = NULL; -+ -+ _mali_osk_list_delinit(&tracker->sync_fence_signal_list); -+ -+ sync_fence = tracker->sync_fence; -+ MALI_DEBUG_ASSERT_POINTER(sync_fence); -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+ fence_status = sync_fence->status; -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ fence_status = atomic_read(&sync_fence->status); -+#else -+ fence_status = sync_fence->fence->ops->signaled(sync_fence->fence); -+#endif -+ -+ system = tracker->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ is_aborting = system->session->is_aborting; -+ if (!is_aborting && (0 > fence_status)) { -+ MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status)); -+ tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT; -+ } -+ -+ waiter = tracker->waiter_sync; -+ MALI_DEBUG_ASSERT_POINTER(waiter); -+ -+ tracker->sync_fence = NULL; -+ tracker->fence.sync_fd = -1; -+ -+ schedule_mask |= mali_timeline_system_release_waiter(system, waiter); -+ -+ /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */ -+ if (is_aborting) { -+ _mali_osk_wait_queue_wake_up(system->wait_queue); -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ /* -+ * Older versions of Linux, before 3.5, doesn't support fput() in interrupt -+ * context. For those older kernels, allocate a list object and put the -+ * fence object on that and defer the call to sync_fence_put() to a workqueue. -+ */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) -+ { -+ struct mali_deferred_fence_put_entry *obj; -+ -+ obj = kzalloc(sizeof(struct mali_deferred_fence_put_entry), GFP_ATOMIC); -+ if (obj) { -+ unsigned long flags; -+ mali_bool schedule = MALI_FALSE; -+ -+ obj->fence = sync_fence; -+ -+ spin_lock_irqsave(&mali_timeline_sync_fence_to_free_lock, flags); -+ if (hlist_empty(&mali_timeline_sync_fence_to_free_list)) -+ schedule = MALI_TRUE; -+ hlist_add_head(&obj->list, &mali_timeline_sync_fence_to_free_list); -+ spin_unlock_irqrestore(&mali_timeline_sync_fence_to_free_lock, flags); -+ -+ if (schedule) -+ schedule_delayed_work(&delayed_sync_fence_put, 0); -+ } -+ } -+#else -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence_put(sync_fence); -+#else -+ fput(sync_fence->file); -+#endif -+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0) */ -+ -+ if (!is_aborting) { -+ mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); -+ } -+ } -+} -+#endif -+_mali_osk_errcode_t mali_timeline_initialize(void) -+{ -+ _mali_osk_atomic_init(&gp_tracker_count, 0); -+ _mali_osk_atomic_init(&phy_pp_tracker_count, 0); -+ _mali_osk_atomic_init(&virt_pp_tracker_count, 0); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ sync_fence_callback_list_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); -+ if (NULL == sync_fence_callback_list_lock) { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ sync_fence_callback_work_t = _mali_osk_wq_create_work( -+ mali_timeline_do_sync_fence_callback, NULL); -+ -+ if (NULL == sync_fence_callback_work_t) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+#endif -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+void mali_timeline_terminate(void) -+{ -+ _mali_osk_atomic_term(&gp_tracker_count); -+ _mali_osk_atomic_term(&phy_pp_tracker_count); -+ _mali_osk_atomic_term(&virt_pp_tracker_count); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (NULL != sync_fence_callback_list_lock) { -+ _mali_osk_spinlock_irq_term(sync_fence_callback_list_lock); -+ sync_fence_callback_list_lock = NULL; -+ } -+ -+ if (NULL != sync_fence_callback_work_t) { -+ _mali_osk_wq_delete_work(sync_fence_callback_work_t); -+ sync_fence_callback_work_t = NULL; -+ } -+#endif -+} -+ -+#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) -+ -+static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id) -+{ -+ struct mali_timeline *timeline; -+ struct mali_timeline_system *system; -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker->timeline); -+ timeline = tracker->timeline; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline->system); -+ system = timeline->system; -+ -+ if (MALI_TIMELINE_MAX > id) { -+ if (MALI_TIMELINE_NO_POINT != tracker->fence.points[id]) { -+ return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]); -+ } else { -+ return MALI_FALSE; -+ } -+ } else { -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id); -+ return MALI_FALSE; -+ } -+} -+ -+static const char *timeline_id_to_string(enum mali_timeline_id id) -+{ -+ switch (id) { -+ case MALI_TIMELINE_GP: -+ return "GP"; -+ case MALI_TIMELINE_PP: -+ return "PP"; -+ case MALI_TIMELINE_SOFT: -+ return "SOFT"; -+ default: -+ return "NONE"; -+ } -+} -+ -+static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type) -+{ -+ switch (type) { -+ case MALI_TIMELINE_TRACKER_GP: -+ return "GP"; -+ case MALI_TIMELINE_TRACKER_PP: -+ return "PP"; -+ case MALI_TIMELINE_TRACKER_SOFT: -+ return "SOFT"; -+ case MALI_TIMELINE_TRACKER_WAIT: -+ return "WAIT"; -+ case MALI_TIMELINE_TRACKER_SYNC: -+ return "SYNC"; -+ default: -+ return "INVALID"; -+ } -+} -+ -+mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker) -+{ -+ struct mali_timeline *timeline = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ timeline = tracker->timeline; -+ -+ if (0 != tracker->trigger_ref_count) { -+ return MALI_TIMELINE_TS_WAITING; -+ } -+ -+ if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) { -+ return MALI_TIMELINE_TS_ACTIVE; -+ } -+ -+ if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) { -+ return MALI_TIMELINE_TS_INIT; -+ } -+ -+ return MALI_TIMELINE_TS_FINISH; -+} -+ -+void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx) -+{ -+ const char *tracker_state = "IWAF"; -+ char state_char = 'I'; -+ char tracker_type[32] = {0}; -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); -+ _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (0 != tracker->trigger_ref_count) { -+ if (print_ctx) -+ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], -+ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)); -+ else -+ MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], -+ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job))); -+ } else { -+ if (print_ctx) -+ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, -+ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job)); -+ else -+ MALI_DEBUG_PRINT(2, ("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, -+ tracker->fence.sync_fd, (unsigned int)(uintptr_t)(tracker->sync_fence), (unsigned int)(uintptr_t)(tracker->job))); -+ -+ } -+#else -+ if (0 != tracker->trigger_ref_count) { -+ if (print_ctx) -+ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], -+ (unsigned int)(uintptr_t)(tracker->job)); -+ else -+ MALI_DEBUG_PRINT(2, ("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], -+ (unsigned int)(uintptr_t)(tracker->job))); -+ } else { -+ if (print_ctx) -+ _mali_osk_ctxprintf(print_ctx, "TL: %s %u %c job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, -+ (unsigned int)(uintptr_t)(tracker->job)); -+ else -+ MALI_DEBUG_PRINT(2, ("TL: %s %u %c job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, -+ (unsigned int)(uintptr_t)(tracker->job))); -+ -+ } -+#endif -+} -+ -+void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx) -+{ -+ struct mali_timeline_tracker *tracker = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ tracker = timeline->tracker_tail; -+ while (NULL != tracker) { -+ mali_timeline_debug_print_tracker(tracker, print_ctx); -+ tracker = tracker->timeline_next; -+ } -+} -+ -+#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) -+void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker) -+{ -+ const char *tracker_state = "IWAF"; -+ char state_char = 'I'; -+ char tracker_type[32] = {0}; -+ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ state_char = *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)); -+ _mali_osk_snprintf(tracker_type, sizeof(tracker_type), "%s", timeline_tracker_type_to_string(tracker->type)); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (0 != tracker->trigger_ref_count) { -+ MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u), fd:%d, fence:(0x%08X)] job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], -+ tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); -+ } else { -+ MALI_PRINT(("TL: %s %u %c fd:%d fence:(0x%08X) job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, -+ tracker->fence.sync_fd, tracker->sync_fence, tracker->job)); -+ } -+#else -+ if (0 != tracker->trigger_ref_count) { -+ MALI_PRINT(("TL: %s %u %c - ref_wait:%u [%s(%u),%s(%u),%s(%u)] job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, tracker->trigger_ref_count, -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "WaitGP" : " ", tracker->fence.points[0], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "WaitPP" : " ", tracker->fence.points[1], -+ is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "WaitSOFT" : " ", tracker->fence.points[2], -+ tracker->job)); -+ } else { -+ MALI_PRINT(("TL: %s %u %c job:(0x%08X)\n", -+ tracker_type, tracker->point, state_char, -+ tracker->job)); -+ } -+#endif -+} -+ -+void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline) -+{ -+ struct mali_timeline_tracker *tracker = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ tracker = timeline->tracker_tail; -+ while (NULL != tracker) { -+ mali_timeline_debug_direct_print_tracker(tracker); -+ tracker = tracker->timeline_next; -+ } -+} -+ -+#endif -+ -+void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx) -+{ -+ int i; -+ int num_printed = 0; -+ u32 tid = _mali_osk_get_tid(); -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ /* Print all timelines */ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ struct mali_timeline *timeline = system->timelines[i]; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ if (NULL == timeline->tracker_head) continue; -+ if (print_ctx) -+ _mali_osk_ctxprintf(print_ctx, "TL: Timeline %s:\n", -+ timeline_id_to_string((enum mali_timeline_id)i)); -+ else -+ MALI_DEBUG_PRINT(2, ("TL: Timeline %s: oldest (%u) next(%u)\n", -+ timeline_id_to_string((enum mali_timeline_id)i), timeline->point_oldest, timeline->point_next)); -+ -+ mali_timeline_debug_print_timeline(timeline, print_ctx); -+ num_printed++; -+ } -+ -+ if (0 == num_printed) { -+ if (print_ctx) -+ _mali_osk_ctxprintf(print_ctx, "TL: All timelines empty\n"); -+ else -+ MALI_DEBUG_PRINT(2, ("TL: All timelines empty\n")); -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+} -+ -+#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+void mali_timeline_dma_fence_callback(void *pp_job_ptr) -+{ -+ struct mali_timeline_system *system; -+ struct mali_timeline_waiter *waiter; -+ struct mali_timeline_tracker *tracker; -+ struct mali_pp_job *pp_job = (struct mali_pp_job *)pp_job_ptr; -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ u32 tid = _mali_osk_get_tid(); -+ mali_bool is_aborting = MALI_FALSE; -+ -+ MALI_DEBUG_ASSERT_POINTER(pp_job); -+ -+ tracker = &pp_job->tracker; -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ -+ system = tracker->system; -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(system->session); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ waiter = tracker->waiter_dma_fence; -+ MALI_DEBUG_ASSERT_POINTER(waiter); -+ -+ schedule_mask |= mali_timeline_system_release_waiter(system, waiter); -+ -+ is_aborting = system->session->is_aborting; -+ -+ /* If aborting, wake up sleepers that are waiting for dma fence callbacks to complete. */ -+ if (is_aborting) { -+ _mali_osk_wait_queue_wake_up(system->wait_queue); -+ } -+ -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+ if (!is_aborting) { -+ mali_executor_schedule_from_mask(schedule_mask, MALI_TRUE); -+ } -+} -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h -new file mode 100755 -index 000000000..3e8bfc8fb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline.h -@@ -0,0 +1,587 @@ -+/* -+ * Copyright (C) 2013-2018 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_TIMELINE_H__ -+#define __MALI_TIMELINE_H__ -+ -+#include "mali_osk.h" -+#include "mali_ukk.h" -+#include "mali_session.h" -+#include "mali_kernel_common.h" -+#include "mali_spinlock_reentrant.h" -+#include "mali_sync.h" -+#include "mali_scheduler_types.h" -+#include -+ -+/** -+ * Soft job timeout. -+ * -+ * Soft jobs have to be signaled as complete after activation. Normally this is done by user space, -+ * but in order to guarantee that every soft job is completed, we also have a timer. -+ */ -+#define MALI_TIMELINE_TIMEOUT_HZ ((unsigned long) (HZ * 3 / 2)) /* 1500 ms. */ -+ -+/** -+ * Timeline type. -+ */ -+typedef enum mali_timeline_id { -+ MALI_TIMELINE_GP = MALI_UK_TIMELINE_GP, /**< GP job timeline. */ -+ MALI_TIMELINE_PP = MALI_UK_TIMELINE_PP, /**< PP job timeline. */ -+ MALI_TIMELINE_SOFT = MALI_UK_TIMELINE_SOFT, /**< Soft job timeline. */ -+ MALI_TIMELINE_MAX = MALI_UK_TIMELINE_MAX -+} mali_timeline_id; -+ -+/** -+ * Used by trackers that should not be added to a timeline (@ref mali_timeline_system_add_tracker). -+ */ -+#define MALI_TIMELINE_NONE MALI_TIMELINE_MAX -+ -+/** -+ * Tracker type. -+ */ -+typedef enum mali_timeline_tracker_type { -+ MALI_TIMELINE_TRACKER_GP = 0, /**< Tracker used by GP jobs. */ -+ MALI_TIMELINE_TRACKER_PP = 1, /**< Tracker used by PP jobs. */ -+ MALI_TIMELINE_TRACKER_SOFT = 2, /**< Tracker used by soft jobs. */ -+ MALI_TIMELINE_TRACKER_WAIT = 3, /**< Tracker used for fence wait. */ -+ MALI_TIMELINE_TRACKER_SYNC = 4, /**< Tracker used for sync fence. */ -+ MALI_TIMELINE_TRACKER_MAX = 5, -+} mali_timeline_tracker_type; -+ -+/** -+ * Tracker activation error. -+ */ -+typedef u32 mali_timeline_activation_error; -+#define MALI_TIMELINE_ACTIVATION_ERROR_NONE 0 -+#define MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT (1<<1) -+#define MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT (1<<0) -+ -+/** -+ * Type used to represent a point on a timeline. -+ */ -+typedef u32 mali_timeline_point; -+ -+/** -+ * Used to represent that no point on a timeline. -+ */ -+#define MALI_TIMELINE_NO_POINT ((mali_timeline_point) 0) -+ -+/** -+ * The maximum span of points on a timeline. A timeline will be considered full if the difference -+ * between the oldest and newest points is equal or larger to this value. -+ */ -+#define MALI_TIMELINE_MAX_POINT_SPAN 65536 -+ -+/** -+ * Magic value used to assert on validity of trackers. -+ */ -+#define MALI_TIMELINE_TRACKER_MAGIC 0xabcdabcd -+ -+struct mali_timeline; -+struct mali_timeline_waiter; -+struct mali_timeline_tracker; -+ -+/** -+ * Timeline fence. -+ */ -+struct mali_timeline_fence { -+ mali_timeline_point points[MALI_TIMELINE_MAX]; /**< For each timeline, a point or MALI_TIMELINE_NO_POINT. */ -+ s32 sync_fd; /**< A file descriptor representing a sync fence, or -1. */ -+}; -+ -+/** -+ * Timeline system. -+ * -+ * The Timeline system has a set of timelines associated with a session. -+ */ -+struct mali_timeline_system { -+ struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ -+ struct mali_timeline *timelines[MALI_TIMELINE_MAX]; /**< The timelines in this system */ -+ -+ /* Single-linked list of unused waiter objects. Uses the tracker_next field in tracker. */ -+ struct mali_timeline_waiter *waiter_empty_list; -+ -+ struct mali_session_data *session; /**< Session that owns this system. */ -+ -+ mali_bool timer_enabled; /**< Set to MALI_TRUE if soft job timer should be enabled, MALI_FALSE if not. */ -+ -+ _mali_osk_wait_queue_t *wait_queue; /**< Wait queue. */ -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ -+#else -+ struct mali_internal_sync_timeline *signaled_sync_tl; /**< Special sync timeline used to create pre-signaled sync fences */ -+#endif -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+}; -+ -+/** -+ * Timeline. Each Timeline system will have MALI_TIMELINE_MAX timelines. -+ */ -+struct mali_timeline { -+ mali_timeline_point point_next; /**< The next available point. */ -+ mali_timeline_point point_oldest; /**< The oldest point not released. */ -+ -+ /* Double-linked list of trackers. Sorted in ascending order by tracker->time_number with -+ * tail pointing to the tracker with the oldest time. */ -+ struct mali_timeline_tracker *tracker_head; -+ struct mali_timeline_tracker *tracker_tail; -+ -+ /* Double-linked list of waiters. Sorted in ascending order by waiter->time_number_wait -+ * with tail pointing to the waiter with oldest wait time. */ -+ struct mali_timeline_waiter *waiter_head; -+ struct mali_timeline_waiter *waiter_tail; -+ -+ struct mali_timeline_system *system; /**< Timeline system this timeline belongs to. */ -+ enum mali_timeline_id id; /**< Timeline type. */ -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_timeline *sync_tl; /**< Sync timeline that corresponds to this timeline. */ -+#else -+ struct mali_internal_sync_timeline *sync_tl; -+#endif -+ mali_bool destroyed; -+ struct mali_spinlock_reentrant *spinlock; /**< Spin lock protecting the timeline system */ -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ /* The following fields are used to time out soft job trackers. */ -+ _mali_osk_wq_delayed_work_t *delayed_work; -+ mali_bool timer_active; -+}; -+ -+/** -+ * Timeline waiter. -+ */ -+struct mali_timeline_waiter { -+ mali_timeline_point point; /**< Point on timeline we are waiting for to be released. */ -+ struct mali_timeline_tracker *tracker; /**< Tracker that is waiting. */ -+ -+ struct mali_timeline_waiter *timeline_next; /**< Next waiter on timeline's waiter list. */ -+ struct mali_timeline_waiter *timeline_prev; /**< Previous waiter on timeline's waiter list. */ -+ -+ struct mali_timeline_waiter *tracker_next; /**< Next waiter on tracker's waiter list. */ -+}; -+ -+/** -+ * Timeline tracker. -+ */ -+struct mali_timeline_tracker { -+ MALI_DEBUG_CODE(u32 magic); /**< Should always be MALI_TIMELINE_TRACKER_MAGIC for a valid tracker. */ -+ -+ mali_timeline_point point; /**< Point on timeline for this tracker */ -+ -+ struct mali_timeline_tracker *timeline_next; /**< Next tracker on timeline's tracker list */ -+ struct mali_timeline_tracker *timeline_prev; /**< Previous tracker on timeline's tracker list */ -+ -+ u32 trigger_ref_count; /**< When zero tracker will be activated */ -+ mali_timeline_activation_error activation_error; /**< Activation error. */ -+ struct mali_timeline_fence fence; /**< Fence used to create this tracker */ -+ -+ /* Single-linked list of waiters. Sorted in order of insertions with -+ * tail pointing to first waiter. */ -+ struct mali_timeline_waiter *waiter_head; -+ struct mali_timeline_waiter *waiter_tail; -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ /* These are only used if the tracker is waiting on a sync fence. */ -+ struct mali_timeline_waiter *waiter_sync; /**< A direct pointer to timeline waiter representing sync fence. */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ -+ struct sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ -+#else -+ struct mali_internal_sync_fence_waiter sync_fence_waiter; /**< Used to connect sync fence and tracker in sync fence wait callback. */ -+ struct mali_internal_sync_fence *sync_fence; /**< The sync fence this tracker is waiting on. */ -+#endif -+ _mali_osk_list_t sync_fence_cancel_list; /**< List node used to cancel sync fence waiters. */ -+ _mali_osk_list_t sync_fence_signal_list; /** < List node used to singal sync fence callback function. */ -+ -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+ struct mali_timeline_waiter *waiter_dma_fence; /**< A direct pointer to timeline waiter representing dma fence. */ -+#endif -+ -+ struct mali_timeline_system *system; /**< Timeline system. */ -+ struct mali_timeline *timeline; /**< Timeline, or NULL if not on a timeline. */ -+ enum mali_timeline_tracker_type type; /**< Type of tracker. */ -+ void *job; /**< Owner of tracker. */ -+ -+ /* The following fields are used to time out soft job trackers. */ -+ unsigned long os_tick_create; -+ unsigned long os_tick_activate; -+ mali_bool timer_active; -+}; -+ -+extern _mali_osk_atomic_t gp_tracker_count; -+extern _mali_osk_atomic_t phy_pp_tracker_count; -+extern _mali_osk_atomic_t virt_pp_tracker_count; -+ -+/** -+ * What follows is a set of functions to check the state of a timeline and to determine where on a -+ * timeline a given point is. Most of these checks will translate the timeline so the oldest point -+ * on the timeline is aligned with zero. Remember that all of these calculation are done on -+ * unsigned integers. -+ * -+ * The following example illustrates the three different states a point can be in. The timeline has -+ * been translated to put the oldest point at zero: -+ * -+ * -+ * -+ * [ point is in forbidden zone ] -+ * 64k wide -+ * MALI_TIMELINE_MAX_POINT_SPAN -+ * -+ * [ point is on timeline ) ( point is released ] -+ * -+ * 0--------------------------##############################--------------------2^32 - 1 -+ * ^ ^ -+ * \ | -+ * oldest point on timeline | -+ * \ -+ * next point on timeline -+ */ -+ -+/** -+ * Compare two timeline points -+ * -+ * Returns true if a is after b, false if a is before or equal to b. -+ * -+ * This funcion ignores MALI_TIMELINE_MAX_POINT_SPAN. Wrapping is supported and -+ * the result will be correct if the points is less then UINT_MAX/2 apart. -+ * -+ * @param a Point on timeline -+ * @param b Point on timeline -+ * @return MALI_TRUE if a is after b -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_point_after(mali_timeline_point a, mali_timeline_point b) -+{ -+ return 0 > ((s32)b) - ((s32)a); -+} -+ -+/** -+ * Check if a point is on timeline. A point is on a timeline if it is greater than, or equal to, -+ * the oldest point, and less than the next point. -+ * -+ * @param timeline Timeline. -+ * @param point Point on timeline. -+ * @return MALI_TRUE if point is on timeline, MALI_FALSE if not. -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_is_point_on(struct mali_timeline *timeline, mali_timeline_point point) -+{ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); -+ -+ return (point - timeline->point_oldest) < (timeline->point_next - timeline->point_oldest); -+} -+ -+/** -+ * Check if a point has been released. A point is released if it is older than the oldest point on -+ * the timeline, newer than the next point, and also not in the forbidden zone. -+ * -+ * @param timeline Timeline. -+ * @param point Point on timeline. -+ * @return MALI_TRUE if point has been release, MALI_FALSE if not. -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_is_point_released(struct mali_timeline *timeline, mali_timeline_point point) -+{ -+ mali_timeline_point point_normalized; -+ mali_timeline_point next_normalized; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); -+ -+ point_normalized = point - timeline->point_oldest; -+ next_normalized = timeline->point_next - timeline->point_oldest; -+ -+ return point_normalized > (next_normalized + MALI_TIMELINE_MAX_POINT_SPAN); -+} -+ -+/** -+ * Check if the tracker that the point relate to has been released. A point is released if the tracker is not on the timeline. -+ * @param timeline Timeline. -+ * @param point Point on timeline. -+ * @return MALI_TRUE if the tracker has been release, MALI_FALSE if not. -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_is_tracker_released(struct mali_timeline *timeline, mali_timeline_point point) -+{ -+ struct mali_timeline_tracker *tracker; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); -+ -+ tracker = timeline->tracker_tail; -+ -+ while (NULL != tracker) { -+ if (point == tracker->point) -+ return MALI_FALSE; -+ tracker = tracker->timeline_next; -+ } -+ -+ return MALI_TRUE; -+} -+ -+/** -+ * Check if a point is valid. A point is valid if is on the timeline or has been released. -+ * -+ * @param timeline Timeline. -+ * @param point Point on timeline. -+ * @return MALI_TRUE if point is valid, MALI_FALSE if not. -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_is_point_valid(struct mali_timeline *timeline, mali_timeline_point point) -+{ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ return mali_timeline_is_point_on(timeline, point) || mali_timeline_is_point_released(timeline, point); -+} -+ -+/** -+ * Check if timeline is empty (has no points on it). A timeline is empty if next == oldest. -+ * -+ * @param timeline Timeline. -+ * @return MALI_TRUE if timeline is empty, MALI_FALSE if not. -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_is_empty(struct mali_timeline *timeline) -+{ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ return timeline->point_next == timeline->point_oldest; -+} -+ -+/** -+ * Check if timeline is full. A valid timeline cannot span more than 64k points (@ref -+ * MALI_TIMELINE_MAX_POINT_SPAN). -+ * -+ * @param timeline Timeline. -+ * @return MALI_TRUE if timeline is full, MALI_FALSE if not. -+ */ -+MALI_STATIC_INLINE mali_bool mali_timeline_is_full(struct mali_timeline *timeline) -+{ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ return MALI_TIMELINE_MAX_POINT_SPAN <= (timeline->point_next - timeline->point_oldest); -+} -+ -+/** -+ * Create a new timeline system. -+ * -+ * @param session The session this timeline system will belong to. -+ * @return New timeline system. -+ */ -+struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session); -+ -+/** -+ * Abort timeline system. -+ * -+ * This will release all pending waiters in the timeline system causing all trackers to be -+ * activated. -+ * -+ * @param system Timeline system to abort all jobs from. -+ */ -+void mali_timeline_system_abort(struct mali_timeline_system *system); -+ -+/** -+ * Destroy an empty timeline system. -+ * -+ * @note @ref mali_timeline_system_abort() should be called prior to this function. -+ * -+ * @param system Timeline system to destroy. -+ */ -+void mali_timeline_system_destroy(struct mali_timeline_system *system); -+ -+/** -+ * Stop the soft job timer. -+ * -+ * @param system Timeline system -+ */ -+void mali_timeline_system_stop_timer(struct mali_timeline_system *system); -+ -+/** -+ * Add a tracker to a timeline system and optionally also on a timeline. -+ * -+ * Once added to the timeline system, the tracker is guaranteed to be activated. The tracker can be -+ * activated before this function returns. Thus, it is also possible that the tracker is released -+ * before this function returns, depending on the tracker type. -+ * -+ * @note Tracker must be initialized (@ref mali_timeline_tracker_init) before being added to the -+ * timeline system. -+ * -+ * @param system Timeline system the tracker will be added to. -+ * @param tracker The tracker to be added. -+ * @param timeline_id Id of the timeline the tracker will be added to, or -+ * MALI_TIMELINE_NONE if it should not be added on a timeline. -+ * @return Point on timeline identifying this tracker, or MALI_TIMELINE_NO_POINT if not on timeline. -+ */ -+mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system, -+ struct mali_timeline_tracker *tracker, -+ enum mali_timeline_id timeline_id); -+ -+/** -+ * Get latest point on timeline. -+ * -+ * @param system Timeline system. -+ * @param timeline_id Id of timeline to get latest point from. -+ * @return Latest point on timeline, or MALI_TIMELINE_NO_POINT if the timeline is empty. -+ */ -+mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system, -+ enum mali_timeline_id timeline_id); -+ -+/** -+ * Initialize tracker. -+ * -+ * Must be called before tracker is added to timeline system (@ref mali_timeline_system_add_tracker). -+ * -+ * @param tracker Tracker to initialize. -+ * @param type Type of tracker. -+ * @param fence Fence used to set up dependencies for tracker. -+ * @param job Pointer to job struct this tracker is associated with. -+ */ -+void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker, -+ mali_timeline_tracker_type type, -+ struct mali_timeline_fence *fence, -+ void *job); -+ -+/** -+ * Grab trigger ref count on tracker. -+ * -+ * This will prevent tracker from being activated until the trigger ref count reaches zero. -+ * -+ * @note Tracker must have been initialized (@ref mali_timeline_tracker_init). -+ * -+ * @param system Timeline system. -+ * @param tracker Tracker. -+ */ -+void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker); -+ -+/** -+ * Release trigger ref count on tracker. -+ * -+ * If the trigger ref count reaches zero, the tracker will be activated. -+ * -+ * @param system Timeline system. -+ * @param tracker Tracker. -+ * @param activation_error Error bitmask if activated with error, or MALI_TIMELINE_ACTIVATION_ERROR_NONE if no error. -+ * @return Scheduling bitmask. -+ */ -+mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error); -+ -+/** -+ * Release a tracker from the timeline system. -+ * -+ * This is used to signal that the job being tracker is finished, either due to normal circumstances -+ * (job complete/abort) or due to a timeout. -+ * -+ * We may need to schedule some subsystems after a tracker has been released and the returned -+ * bitmask will tell us if it is necessary. If the return value is non-zero, this value needs to be -+ * sent as an input parameter to @ref mali_scheduler_schedule_from_mask() to do the scheduling. -+ * -+ * @note Tracker must have been activated before being released. -+ * @warning Not calling @ref mali_scheduler_schedule_from_mask() after releasing a tracker can lead -+ * to a deadlock. -+ * -+ * @param tracker Tracker being released. -+ * @return Scheduling bitmask. -+ */ -+mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker); -+ -+MALI_STATIC_INLINE mali_bool mali_timeline_tracker_activation_error( -+ struct mali_timeline_tracker *tracker) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tracker); -+ return (MALI_TIMELINE_ACTIVATION_ERROR_FATAL_BIT & -+ tracker->activation_error) ? MALI_TRUE : MALI_FALSE; -+} -+ -+/** -+ * Copy data from a UK fence to a Timeline fence. -+ * -+ * @param fence Timeline fence. -+ * @param uk_fence UK fence. -+ */ -+void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence); -+ -+_mali_osk_errcode_t mali_timeline_initialize(void); -+ -+void mali_timeline_terminate(void); -+ -+MALI_STATIC_INLINE mali_bool mali_timeline_has_gp_job(void) -+{ -+ return 0 < _mali_osk_atomic_read(&gp_tracker_count); -+} -+ -+MALI_STATIC_INLINE mali_bool mali_timeline_has_physical_pp_job(void) -+{ -+ return 0 < _mali_osk_atomic_read(&phy_pp_tracker_count); -+} -+ -+MALI_STATIC_INLINE mali_bool mali_timeline_has_virtual_pp_job(void) -+{ -+ return 0 < _mali_osk_atomic_read(&virt_pp_tracker_count); -+} -+ -+#if defined(DEBUG) -+#define MALI_TIMELINE_DEBUG_FUNCTIONS -+#endif /* DEBUG */ -+#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) -+ -+/** -+ * Tracker state. Used for debug printing. -+ */ -+typedef enum mali_timeline_tracker_state { -+ MALI_TIMELINE_TS_INIT = 0, -+ MALI_TIMELINE_TS_WAITING = 1, -+ MALI_TIMELINE_TS_ACTIVE = 2, -+ MALI_TIMELINE_TS_FINISH = 3, -+} mali_timeline_tracker_state; -+ -+/** -+ * Get tracker state. -+ * -+ * @param tracker Tracker to check. -+ * @return State of tracker. -+ */ -+mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker); -+ -+/** -+ * Print debug information about tracker. -+ * -+ * @param tracker Tracker to print. -+ */ -+void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker, _mali_osk_print_ctx *print_ctx); -+ -+/** -+ * Print debug information about timeline. -+ * -+ * @param timeline Timeline to print. -+ */ -+void mali_timeline_debug_print_timeline(struct mali_timeline *timeline, _mali_osk_print_ctx *print_ctx); -+ -+#if !(LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0)) -+void mali_timeline_debug_direct_print_tracker(struct mali_timeline_tracker *tracker); -+void mali_timeline_debug_direct_print_timeline(struct mali_timeline *timeline); -+#endif -+ -+/** -+ * Print debug information about timeline system. -+ * -+ * @param system Timeline system to print. -+ */ -+void mali_timeline_debug_print_system(struct mali_timeline_system *system, _mali_osk_print_ctx *print_ctx); -+ -+#endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */ -+ -+#if defined(CONFIG_MALI_DMA_BUF_FENCE) -+/** -+ * The timeline dma fence callback when dma fence signal. -+ * -+ * @param pp_job_ptr The pointer to pp job that link to the signaled dma fence. -+ */ -+void mali_timeline_dma_fence_callback(void *pp_job_ptr); -+#endif -+ -+#endif /* __MALI_TIMELINE_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c -new file mode 100755 -index 000000000..1ab13f509 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.c -@@ -0,0 +1,218 @@ -+/* -+ * Copyright (C) 2013-2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include -+#include "mali_timeline_fence_wait.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_spinlock_reentrant.h" -+ -+/** -+ * Allocate a fence waiter tracker. -+ * -+ * @return New fence waiter if successful, NULL if not. -+ */ -+static struct mali_timeline_fence_wait_tracker *mali_timeline_fence_wait_tracker_alloc(void) -+{ -+ return (struct mali_timeline_fence_wait_tracker *) _mali_osk_calloc(1, sizeof(struct mali_timeline_fence_wait_tracker)); -+} -+ -+/** -+ * Free fence waiter tracker. -+ * -+ * @param wait Fence wait tracker to free. -+ */ -+static void mali_timeline_fence_wait_tracker_free(struct mali_timeline_fence_wait_tracker *wait) -+{ -+ MALI_DEBUG_ASSERT_POINTER(wait); -+ _mali_osk_atomic_term(&wait->refcount); -+ _mali_osk_free(wait); -+} -+ -+/** -+ * Check if fence wait tracker has been activated. Used as a wait queue condition. -+ * -+ * @param data Fence waiter. -+ * @return MALI_TRUE if tracker has been activated, MALI_FALSE if not. -+ */ -+static mali_bool mali_timeline_fence_wait_tracker_is_activated(void *data) -+{ -+ struct mali_timeline_fence_wait_tracker *wait; -+ -+ wait = (struct mali_timeline_fence_wait_tracker *) data; -+ MALI_DEBUG_ASSERT_POINTER(wait); -+ -+ return wait->activated; -+} -+ -+/** -+ * Check if fence has been signaled. -+ * -+ * @param system Timeline system. -+ * @param fence Timeline fence. -+ * @return MALI_TRUE if fence is signaled, MALI_FALSE if not. -+ */ -+static mali_bool mali_timeline_fence_wait_check_status(struct mali_timeline_system *system, struct mali_timeline_fence *fence) -+{ -+ int i; -+ u32 tid = _mali_osk_get_tid(); -+ mali_bool ret = MALI_TRUE; -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence = NULL; -+#else -+ struct mali_internal_sync_fence *sync_fence = NULL; -+#endif -+#endif -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ struct mali_timeline *timeline; -+ mali_timeline_point point; -+ -+ point = fence->points[i]; -+ -+ if (likely(MALI_TIMELINE_NO_POINT == point)) { -+ /* Fence contains no point on this timeline. */ -+ continue; -+ } -+ -+ timeline = system->timelines[i]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ if (unlikely(!mali_timeline_is_point_valid(timeline, point))) { -+ MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n", point, timeline->point_oldest, timeline->point_next)); -+ } -+ -+ if (!mali_timeline_is_point_released(timeline, point)) { -+ ret = MALI_FALSE; -+ goto exit; -+ } -+ } -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (-1 != fence->sync_fd) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence = sync_fence_fdget(fence->sync_fd); -+#else -+ sync_fence = mali_internal_sync_fence_fdget(fence->sync_fd); -+#endif -+ if (likely(NULL != sync_fence)) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+ if (0 == sync_fence->status) { -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ if (0 < atomic_read(&sync_fence->status)) { -+#else -+ if (0 == sync_fence->fence->ops->signaled(sync_fence->fence)) { -+#endif -+ ret = MALI_FALSE; -+ -+ } else { -+ ret = MALI_TRUE; -+ } -+ } else { -+ MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", fence->sync_fd)); -+ } -+ } -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+exit: -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ if (NULL != sync_fence) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence_put(sync_fence); -+#else -+ fput(sync_fence->file); -+#endif -+ } -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ return ret; -+} -+ -+mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout) -+{ -+ struct mali_timeline_fence_wait_tracker *wait; -+ mali_timeline_point point; -+ mali_bool ret; -+ -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ MALI_DEBUG_PRINT(4, ("Mali Timeline: wait on fence\n")); -+ -+ if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY == timeout) { -+ return mali_timeline_fence_wait_check_status(system, fence); -+ } -+ -+ wait = mali_timeline_fence_wait_tracker_alloc(); -+ if (unlikely(NULL == wait)) { -+ MALI_PRINT_ERROR(("Mali Timeline: failed to allocate data for fence wait\n")); -+ return MALI_FALSE; -+ } -+ -+ wait->activated = MALI_FALSE; -+ wait->system = system; -+ -+ /* Initialize refcount to two references. The reference first will be released by this -+ * function after the wait is over. The second reference will be released when the tracker -+ * is activated. */ -+ _mali_osk_atomic_init(&wait->refcount, 2); -+ -+ /* Add tracker to timeline system, but not to a timeline. */ -+ mali_timeline_tracker_init(&wait->tracker, MALI_TIMELINE_TRACKER_WAIT, fence, wait); -+ point = mali_timeline_system_add_tracker(system, &wait->tracker, MALI_TIMELINE_NONE); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); -+ MALI_IGNORE(point); -+ -+ /* Wait for the tracker to be activated or time out. */ -+ if (MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER == timeout) { -+ _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait); -+ } else { -+ _mali_osk_wait_queue_wait_event_timeout(system->wait_queue, mali_timeline_fence_wait_tracker_is_activated, (void *) wait, timeout); -+ } -+ -+ ret = wait->activated; -+ -+ if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { -+ mali_timeline_fence_wait_tracker_free(wait); -+ } -+ -+ return ret; -+} -+ -+void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *wait) -+{ -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ -+ MALI_DEBUG_ASSERT_POINTER(wait); -+ MALI_DEBUG_ASSERT_POINTER(wait->system); -+ -+ MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for fence wait tracker\n")); -+ -+ MALI_DEBUG_ASSERT(MALI_FALSE == wait->activated); -+ wait->activated = MALI_TRUE; -+ -+ _mali_osk_wait_queue_wake_up(wait->system->wait_queue); -+ -+ /* Nothing can wait on this tracker, so nothing to schedule after release. */ -+ schedule_mask = mali_timeline_tracker_release(&wait->tracker); -+ MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); -+ MALI_IGNORE(schedule_mask); -+ -+ if (0 == _mali_osk_atomic_dec_return(&wait->refcount)) { -+ mali_timeline_fence_wait_tracker_free(wait); -+ } -+} -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h -new file mode 100755 -index 000000000..9da12baee ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_fence_wait.h -@@ -0,0 +1,67 @@ -+/* -+ * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_timeline_fence_wait.h -+ * -+ * This file contains functions used to wait until a Timeline fence is signaled. -+ */ -+ -+#ifndef __MALI_TIMELINE_FENCE_WAIT_H__ -+#define __MALI_TIMELINE_FENCE_WAIT_H__ -+ -+#include "mali_osk.h" -+#include "mali_timeline.h" -+ -+/** -+ * If used as the timeout argument in @ref mali_timeline_fence_wait, a timer is not used and the -+ * function only returns when the fence is signaled. -+ */ -+#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER ((u32) -1) -+ -+/** -+ * If used as the timeout argument in @ref mali_timeline_fence_wait, the function will return -+ * immediately with the current state of the fence. -+ */ -+#define MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY 0 -+ -+/** -+ * Fence wait tracker. -+ * -+ * The fence wait tracker is added to the Timeline system with the fence we are waiting on as a -+ * dependency. We will then perform a blocking wait, possibly with a timeout, until the tracker is -+ * activated, which happens when the fence is signaled. -+ */ -+struct mali_timeline_fence_wait_tracker { -+ mali_bool activated; /**< MALI_TRUE if the tracker has been activated, MALI_FALSE if not. */ -+ _mali_osk_atomic_t refcount; /**< Reference count. */ -+ struct mali_timeline_system *system; /**< Timeline system. */ -+ struct mali_timeline_tracker tracker; /**< Timeline tracker. */ -+}; -+ -+/** -+ * Wait for a fence to be signaled, or timeout is reached. -+ * -+ * @param system Timeline system. -+ * @param fence Fence to wait on. -+ * @param timeout Timeout in ms, or MALI_TIMELINE_FENCE_WAIT_TIMEOUT_NEVER or -+ * MALI_TIMELINE_FENCE_WAIT_TIMEOUT_IMMEDIATELY. -+ * @return MALI_TRUE if signaled, MALI_FALSE if timed out. -+ */ -+mali_bool mali_timeline_fence_wait(struct mali_timeline_system *system, struct mali_timeline_fence *fence, u32 timeout); -+ -+/** -+ * Used by the Timeline system to activate a fence wait tracker. -+ * -+ * @param fence_wait_tracker Fence waiter tracker. -+ */ -+void mali_timeline_fence_wait_activate(struct mali_timeline_fence_wait_tracker *fence_wait_tracker); -+ -+#endif /* __MALI_TIMELINE_FENCE_WAIT_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c -new file mode 100755 -index 000000000..bb7f6a04e ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.c -@@ -0,0 +1,179 @@ -+/* -+ * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include -+#include "mali_timeline_sync_fence.h" -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_sync.h" -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+/** -+ * Creates a sync fence tracker and a sync fence. Adds sync fence tracker to Timeline system and -+ * returns sync fence. The sync fence will be signaled when the sync fence tracker is activated. -+ * -+ * @param timeline Timeline. -+ * @param point Point on timeline. -+ * @return Sync fence that will be signaled when tracker is activated. -+ */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static struct sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) -+#else -+static struct mali_internal_sync_fence *mali_timeline_sync_fence_create_and_add_tracker(struct mali_timeline *timeline, mali_timeline_point point) -+#endif -+{ -+ struct mali_timeline_sync_fence_tracker *sync_fence_tracker; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence; -+#else -+ struct mali_internal_sync_fence *sync_fence; -+#endif -+ struct mali_timeline_fence fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT != point); -+ -+ /* Allocate sync fence tracker. */ -+ sync_fence_tracker = _mali_osk_calloc(1, sizeof(struct mali_timeline_sync_fence_tracker)); -+ if (NULL == sync_fence_tracker) { -+ MALI_PRINT_ERROR(("Mali Timeline: sync_fence_tracker allocation failed\n")); -+ return NULL; -+ } -+ -+ /* Create sync flag. */ -+ MALI_DEBUG_ASSERT_POINTER(timeline->sync_tl); -+ sync_fence_tracker->flag = mali_sync_flag_create(timeline->sync_tl, point); -+ if (NULL == sync_fence_tracker->flag) { -+ MALI_PRINT_ERROR(("Mali Timeline: sync_flag creation failed\n")); -+ _mali_osk_free(sync_fence_tracker); -+ return NULL; -+ } -+ -+ /* Create sync fence from sync flag. */ -+ sync_fence = mali_sync_flag_create_fence(sync_fence_tracker->flag); -+ if (NULL == sync_fence) { -+ MALI_PRINT_ERROR(("Mali Timeline: sync_fence creation failed\n")); -+ mali_sync_flag_put(sync_fence_tracker->flag); -+ _mali_osk_free(sync_fence_tracker); -+ return NULL; -+ } -+ -+ /* Setup fence for tracker. */ -+ _mali_osk_memset(&fence, 0, sizeof(struct mali_timeline_fence)); -+ fence.sync_fd = -1; -+ fence.points[timeline->id] = point; -+ -+ /* Finally, add the tracker to Timeline system. */ -+ mali_timeline_tracker_init(&sync_fence_tracker->tracker, MALI_TIMELINE_TRACKER_SYNC, &fence, sync_fence_tracker); -+ point = mali_timeline_system_add_tracker(timeline->system, &sync_fence_tracker->tracker, MALI_TIMELINE_NONE); -+ MALI_DEBUG_ASSERT(MALI_TIMELINE_NO_POINT == point); -+ -+ return sync_fence; -+} -+ -+s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence) -+{ -+ u32 i; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence_acc = NULL; -+#else -+ struct mali_internal_sync_fence *sync_fence_acc = NULL; -+#endif -+ MALI_DEBUG_ASSERT_POINTER(system); -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ for (i = 0; i < MALI_TIMELINE_MAX; ++i) { -+ struct mali_timeline *timeline; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence; -+#else -+ struct mali_internal_sync_fence *sync_fence; -+#endif -+ if (MALI_TIMELINE_NO_POINT == fence->points[i]) continue; -+ -+ timeline = system->timelines[i]; -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ sync_fence = mali_timeline_sync_fence_create_and_add_tracker(timeline, fence->points[i]); -+ if (NULL == sync_fence) goto error; -+ -+ if (NULL != sync_fence_acc) { -+ /* Merge sync fences. */ -+ sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); -+ if (NULL == sync_fence_acc) goto error; -+ } else { -+ /* This was the first sync fence created. */ -+ sync_fence_acc = sync_fence; -+ } -+ } -+ -+ if (-1 != fence->sync_fd) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_fence *sync_fence; -+ sync_fence = sync_fence_fdget(fence->sync_fd); -+#else -+ struct mali_internal_sync_fence *sync_fence; -+ sync_fence = mali_internal_sync_fence_fdget(fence->sync_fd); -+#endif -+ -+ if (NULL == sync_fence) goto error; -+ -+ if (NULL != sync_fence_acc) { -+ sync_fence_acc = mali_sync_fence_merge(sync_fence_acc, sync_fence); -+ if (NULL == sync_fence_acc) goto error; -+ } else { -+ sync_fence_acc = sync_fence; -+ } -+ } -+ -+ if (NULL == sync_fence_acc) { -+ MALI_DEBUG_ASSERT_POINTER(system->signaled_sync_tl); -+ -+ /* There was nothing to wait on, so return an already signaled fence. */ -+ -+ sync_fence_acc = mali_sync_timeline_create_signaled_fence(system->signaled_sync_tl); -+ if (NULL == sync_fence_acc) goto error; -+ } -+ -+ /* Return file descriptor for the accumulated sync fence. */ -+ return mali_sync_fence_fd_alloc(sync_fence_acc); -+ -+error: -+ if (NULL != sync_fence_acc) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_fence_put(sync_fence_acc); -+#else -+ fput(sync_fence_acc->file); -+#endif -+ } -+ -+ return -1; -+} -+ -+void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker) -+{ -+ mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker); -+ MALI_DEBUG_ASSERT_POINTER(sync_fence_tracker->flag); -+ -+ MALI_DEBUG_PRINT(4, ("Mali Timeline: activation for sync fence tracker\n")); -+ -+ /* Signal flag and release reference. */ -+ mali_sync_flag_signal(sync_fence_tracker->flag, 0); -+ mali_sync_flag_put(sync_fence_tracker->flag); -+ -+ /* Nothing can wait on this tracker, so nothing to schedule after release. */ -+ schedule_mask = mali_timeline_tracker_release(&sync_fence_tracker->tracker); -+ MALI_DEBUG_ASSERT(MALI_SCHEDULER_MASK_EMPTY == schedule_mask); -+ -+ _mali_osk_free(sync_fence_tracker); -+} -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h -new file mode 100755 -index 000000000..65e368ae7 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_timeline_sync_fence.h -@@ -0,0 +1,51 @@ -+/* -+ * Copyright (C) 2013, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_timeline_sync_fence.h -+ * -+ * This file contains code related to creating sync fences from timeline fences. -+ */ -+ -+#ifndef __MALI_TIMELINE_SYNC_FENCE_H__ -+#define __MALI_TIMELINE_SYNC_FENCE_H__ -+ -+#include "mali_timeline.h" -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ -+/** -+ * Sync fence tracker. -+ */ -+struct mali_timeline_sync_fence_tracker { -+ struct mali_sync_flag *flag; /**< Sync flag used to connect tracker and sync fence. */ -+ struct mali_timeline_tracker tracker; /**< Timeline tracker. */ -+}; -+ -+/** -+ * Create a sync fence that will be signaled when @ref fence is signaled. -+ * -+ * @param system Timeline system. -+ * @param fence Fence to create sync fence from. -+ * @return File descriptor for new sync fence, or -1 on error. -+ */ -+s32 mali_timeline_sync_fence_create(struct mali_timeline_system *system, struct mali_timeline_fence *fence); -+ -+/** -+ * Used by the Timeline system to activate a sync fence tracker. -+ * -+ * @param sync_fence_tracker Sync fence tracker. -+ * -+ */ -+void mali_timeline_sync_fence_activate(struct mali_timeline_sync_fence_tracker *sync_fence_tracker); -+ -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+#endif /* __MALI_TIMELINE_SYNC_FENCE_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_ukk.h b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h -new file mode 100755 -index 000000000..55a05c504 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_ukk.h -@@ -0,0 +1,551 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_ukk.h -+ * Defines the kernel-side interface of the user-kernel interface -+ */ -+ -+#ifndef __MALI_UKK_H__ -+#define __MALI_UKK_H__ -+ -+#include "mali_osk.h" -+#include "mali_uk_types.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/** -+ * @addtogroup uddapi Unified Device Driver (UDD) APIs -+ * -+ * @{ -+ */ -+ -+/** -+ * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs -+ * -+ * - The _mali_uk functions are an abstraction of the interface to the device -+ * driver. On certain OSs, this would be implemented via the IOCTL interface. -+ * On other OSs, it could be via extension of some Device Driver Class, or -+ * direct function call for Bare metal/RTOSs. -+ * - It is important to note that: -+ * - The Device Driver has implemented the _mali_ukk set of functions -+ * - The Base Driver calls the corresponding set of _mali_uku functions. -+ * - What requires porting is solely the calling mechanism from User-side to -+ * Kernel-side, and propagating back the results. -+ * - Each U/K function is associated with a (group, number) pair from -+ * \ref _mali_uk_functions to make it possible for a common function in the -+ * Base Driver and Device Driver to route User/Kernel calls from/to the -+ * correct _mali_uk function. For example, in an IOCTL system, the IOCTL number -+ * would be formed based on the group and number assigned to the _mali_uk -+ * function, as listed in \ref _mali_uk_functions. On the user-side, each -+ * _mali_uku function would just make an IOCTL with the IOCTL-code being an -+ * encoded form of the (group, number) pair. On the kernel-side, the Device -+ * Driver's IOCTL handler decodes the IOCTL-code back into a (group, number) -+ * pair, and uses this to determine which corresponding _mali_ukk should be -+ * called. -+ * - Refer to \ref _mali_uk_functions for more information about this -+ * (group, number) pairing. -+ * - In a system where there is no distinction between user and kernel-side, -+ * the U/K interface may be implemented as:@code -+ * MALI_STATIC_INLINE _mali_osk_errcode_t _mali_uku_examplefunction( _mali_uk_examplefunction_s *args ) -+ * { -+ * return mali_ukk_examplefunction( args ); -+ * } -+ * @endcode -+ * - Therefore, all U/K calls behave \em as \em though they were direct -+ * function calls (but the \b implementation \em need \em not be a direct -+ * function calls) -+ * -+ * @note Naming the _mali_uk functions the same on both User and Kernel sides -+ * on non-RTOS systems causes debugging issues when setting breakpoints. In -+ * this case, it is not clear which function the breakpoint is put on. -+ * Therefore the _mali_uk functions in user space are prefixed with \c _mali_uku -+ * and in kernel space with \c _mali_ukk. The naming for the argument -+ * structures is unaffected. -+ * -+ * - The _mali_uk functions are synchronous. -+ * - Arguments to the _mali_uk functions are passed in a structure. The only -+ * parameter passed to the _mali_uk functions is a pointer to this structure. -+ * This first member of this structure, ctx, is a pointer to a context returned -+ * by _mali_uku_open(). For example:@code -+ * typedef struct -+ * { -+ * void *ctx; -+ * u32 number_of_cores; -+ * } _mali_uk_get_gp_number_of_cores_s; -+ * @endcode -+ * -+ * - Each _mali_uk function has its own argument structure named after the -+ * function. The argument is distinguished by the _s suffix. -+ * - The argument types are defined by the base driver and user-kernel -+ * interface. -+ * - All _mali_uk functions return a standard \ref _mali_osk_errcode_t. -+ * - Only arguments of type input or input/output need be initialized before -+ * calling a _mali_uk function. -+ * - Arguments of type output and input/output are only valid when the -+ * _mali_uk function returns \ref _MALI_OSK_ERR_OK. -+ * - The \c ctx member is always invalid after it has been used by a -+ * _mali_uk function, except for the context management functions -+ * -+ * -+ * \b Interface \b restrictions -+ * -+ * The requirements of the interface mean that an implementation of the -+ * User-kernel interface may do no 'real' work. For example, the following are -+ * illegal in the User-kernel implementation: -+ * - Calling functions necessary for operation on all systems, which would -+ * not otherwise get called on RTOS systems. -+ * - For example, a U/K interface that calls multiple _mali_ukk functions -+ * during one particular U/K call. This could not be achieved by the same code -+ * which uses direct function calls for the U/K interface. -+ * - Writing in values to the args members, when otherwise these members would -+ * not hold a useful value for a direct function call U/K interface. -+ * - For example, U/K interface implementation that take NULL members in -+ * their arguments structure from the user side, but those members are -+ * replaced with non-NULL values in the kernel-side of the U/K interface -+ * implementation. A scratch area for writing data is one such example. In this -+ * case, a direct function call U/K interface would segfault, because no code -+ * would be present to replace the NULL pointer with a meaningful pointer. -+ * - Note that we discourage the case where the U/K implementation changes -+ * a NULL argument member to non-NULL, and then the Device Driver code (outside -+ * of the U/K layer) re-checks this member for NULL, and corrects it when -+ * necessary. Whilst such code works even on direct function call U/K -+ * intefaces, it reduces the testing coverage of the Device Driver code. This -+ * is because we have no way of testing the NULL == value path on an OS -+ * implementation. -+ * -+ * A number of allowable examples exist where U/K interfaces do 'real' work: -+ * - The 'pointer switching' technique for \ref _mali_ukk_get_system_info -+ * - In this case, without the pointer switching on direct function call -+ * U/K interface, the Device Driver code still sees the same thing: a pointer -+ * to which it can write memory. This is because such a system has no -+ * distinction between a user and kernel pointer. -+ * - Writing an OS-specific value into the ukk_private member for -+ * _mali_ukk_mem_mmap(). -+ * - In this case, this value is passed around by Device Driver code, but -+ * its actual value is never checked. Device Driver code simply passes it from -+ * the U/K layer to the OSK layer, where it can be acted upon. In this case, -+ * \em some OS implementations of the U/K (_mali_ukk_mem_mmap()) and OSK -+ * (_mali_osk_mem_mapregion_init()) functions will collaborate on the -+ * meaning of ukk_private member. On other OSs, it may be unused by both -+ * U/K and OSK layers -+ * - Therefore, on error inside the U/K interface implementation itself, -+ * it will be as though the _mali_ukk function itself had failed, and cleaned -+ * up after itself. -+ * - Compare this to a direct function call U/K implementation, where all -+ * error cleanup is handled by the _mali_ukk function itself. The direct -+ * function call U/K interface implementation is automatically atomic. -+ * -+ * The last example highlights a consequence of all U/K interface -+ * implementations: they must be atomic with respect to the Device Driver code. -+ * And therefore, should Device Driver code succeed but the U/K implementation -+ * fail afterwards (but before return to user-space), then the U/K -+ * implementation must cause appropriate cleanup actions to preserve the -+ * atomicity of the interface. -+ * -+ * @{ -+ */ -+ -+ -+/** @defgroup _mali_uk_context U/K Context management -+ * -+ * These functions allow for initialisation of the user-kernel interface once per process. -+ * -+ * Generally the context will store the OS specific object to communicate with the kernel device driver and further -+ * state information required by the specific implementation. The context is shareable among all threads in the caller process. -+ * -+ * On IOCTL systems, this is likely to be a file descriptor as a result of opening the kernel device driver. -+ * -+ * On a bare-metal/RTOS system with no distinction between kernel and -+ * user-space, the U/K interface simply calls the _mali_ukk variant of the -+ * function by direct function call. In this case, the context returned is the -+ * mali_session_data from _mali_ukk_open(). -+ * -+ * The kernel side implementations of the U/K interface expect the first member of the argument structure to -+ * be the context created by _mali_uku_open(). On some OS implementations, the meaning of this context -+ * will be different between user-side and kernel-side. In which case, the kernel-side will need to replace this context -+ * with the kernel-side equivalent, because user-side will not have access to kernel-side data. The context parameter -+ * in the argument structure therefore has to be of type input/output. -+ * -+ * It should be noted that the caller cannot reuse the \c ctx member of U/K -+ * argument structure after a U/K call, because it may be overwritten. Instead, -+ * the context handle must always be stored elsewhere, and copied into -+ * the appropriate U/K argument structure for each user-side call to -+ * the U/K interface. This is not usually a problem, since U/K argument -+ * structures are usually placed on the stack. -+ * -+ * @{ */ -+ -+/** @brief Begin a new Mali Device Driver session -+ * -+ * This is used to obtain a per-process context handle for all future U/K calls. -+ * -+ * @param context pointer to storage to return a (void*)context handle. -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_open(void **context); -+ -+/** @brief End a Mali Device Driver session -+ * -+ * This should be called when the process no longer requires use of the Mali Device Driver. -+ * -+ * The context handle must not be used after it has been closed. -+ * -+ * @param context pointer to a stored (void*)context handle. -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_close(void **context); -+ -+/** @} */ /* end group _mali_uk_context */ -+ -+ -+/** @addtogroup _mali_uk_core U/K Core -+ * -+ * The core functions provide the following functionality: -+ * - verify that the user and kernel API are compatible -+ * - retrieve information about the cores and memory banks in the system -+ * - wait for the result of jobs started on a core -+ * -+ * @{ */ -+ -+/** @brief Waits for a job notification. -+ * -+ * Sleeps until notified or a timeout occurs. Returns information about the notification. -+ * -+ * @param args see _mali_uk_wait_for_notification_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args); -+ -+/** @brief Post a notification to the notification queue of this application. -+ * -+ * @param args see _mali_uk_post_notification_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args); -+ -+/** @brief Verifies if the user and kernel side of this API are compatible. -+ * -+ * This function is obsolete, but kept to allow old, incompatible user space -+ * clients to robustly detect the incompatibility. -+ * -+ * @param args see _mali_uk_get_api_version_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args); -+ -+/** @brief Verifies if the user and kernel side of this API are compatible. -+ * -+ * @param args see _mali_uk_get_api_version_v2_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args); -+ -+/** @brief Get the user space settings applicable for calling process. -+ * -+ * @param args see _mali_uk_get_user_settings_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args); -+ -+/** @brief Get a user space setting applicable for calling process. -+ * -+ * @param args see _mali_uk_get_user_setting_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args); -+ -+/* @brief Grant or deny high priority scheduling for this session. -+ * -+ * @param args see _mali_uk_request_high_priority_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args); -+ -+/** @brief Make process sleep if the pending big job in kernel >= MALI_MAX_PENDING_BIG_JOB -+ * -+ */ -+_mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args); -+ -+/** @} */ /* end group _mali_uk_core */ -+ -+ -+/** @addtogroup _mali_uk_memory U/K Memory -+ * -+ * The memory functions provide functionality with and without a Mali-MMU present. -+ * -+ * For Mali-MMU based systems, the following functionality is provided: -+ * - Initialize and terminate MALI virtual address space -+ * - Allocate/deallocate physical memory to a MALI virtual address range and map into/unmap from the -+ * current process address space -+ * - Map/unmap external physical memory into the MALI virtual address range -+ * -+ * For Mali-nonMMU based systems: -+ * - Allocate/deallocate MALI memory -+ * -+ * @{ */ -+ -+/** @brief Map Mali Memory into the current user process -+ * -+ * Maps Mali memory into the current user process in a generic way. -+ * -+ * This function is to be used for Mali-MMU mode. The function is available in both Mali-MMU and Mali-nonMMU modes, -+ * but should not be called by a user process in Mali-nonMMU mode. -+ * -+ * The implementation and operation of _mali_ukk_mem_mmap() is dependant on whether the driver is built for Mali-MMU -+ * or Mali-nonMMU: -+ * - In the nonMMU case, _mali_ukk_mem_mmap() requires a physical address to be specified. For this reason, an OS U/K -+ * implementation should not allow this to be called from user-space. In any case, nonMMU implementations are -+ * inherently insecure, and so the overall impact is minimal. Mali-MMU mode should be used if security is desired. -+ * - In the MMU case, _mali_ukk_mem_mmap() the _mali_uk_mem_mmap_s::phys_addr -+ * member is used for the \em Mali-virtual address desired for the mapping. The -+ * implementation of _mali_ukk_mem_mmap() will allocate both the CPU-virtual -+ * and CPU-physical addresses, and can cope with mapping a contiguous virtual -+ * address range to a sequence of non-contiguous physical pages. In this case, -+ * the CPU-physical addresses are not communicated back to the user-side, as -+ * they are unnecsessary; the \em Mali-virtual address range must be used for -+ * programming Mali structures. -+ * -+ * In the second (MMU) case, _mali_ukk_mem_mmap() handles management of -+ * CPU-virtual and CPU-physical ranges, but the \em caller must manage the -+ * \em Mali-virtual address range from the user-side. -+ * -+ * @note Mali-virtual address ranges are entirely separate between processes. -+ * It is not possible for a process to accidentally corrupt another process' -+ * \em Mali-virtual address space. -+ * -+ * @param args see _mali_uk_mem_mmap_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_mem_mmap(_mali_uk_mem_mmap_s *args); -+ -+/** @brief Unmap Mali Memory from the current user process -+ * -+ * Unmaps Mali memory from the current user process in a generic way. This only operates on Mali memory supplied -+ * from _mali_ukk_mem_mmap(). -+ * -+ * @param args see _mali_uk_mem_munmap_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_mem_munmap(_mali_uk_mem_munmap_s *args); -+ -+/** @brief Determine the buffer size necessary for an MMU page table dump. -+ * @param args see _mali_uk_query_mmu_page_table_dump_size_s in mali_utgard_uk_types.h -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args); -+/** @brief Dump MMU Page tables. -+ * @param args see _mali_uk_dump_mmu_page_table_s in mali_utgard_uk_types.h -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args); -+ -+/** @brief Write user data to specified Mali memory without causing segfaults. -+ * @param args see _mali_uk_mem_write_safe_s in mali_utgard_uk_types.h -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args); -+ -+/** @} */ /* end group _mali_uk_memory */ -+ -+ -+/** @addtogroup _mali_uk_pp U/K Fragment Processor -+ * -+ * The Fragment Processor (aka PP (Pixel Processor)) functions provide the following functionality: -+ * - retrieving version of the fragment processors -+ * - determine number of fragment processors -+ * - starting a job on a fragment processor -+ * -+ * @{ */ -+ -+/** @brief Issue a request to start a new job on a Fragment Processor. -+ * -+ * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can -+ * try to start the job again. -+ * -+ * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job -+ * which the hardware hasn't actually started processing yet. In this case the new job will be started instead and the -+ * existing one returned, otherwise the new job is started and the status field args->status is set to -+ * _MALI_UK_START_JOB_STARTED. -+ * -+ * Job completion can be awaited with _mali_ukk_wait_for_notification(). -+ * -+ * @param ctx user-kernel context (mali_session) -+ * @param uargs see _mali_uk_pp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, _mali_uk_pp_start_job_s *uargs); -+ -+/** -+ * @brief Issue a request to start new jobs on both Vertex Processor and Fragment Processor. -+ * -+ * @note Will call into @ref _mali_ukk_pp_start_job and @ref _mali_ukk_gp_start_job. -+ * -+ * @param ctx user-kernel context (mali_session) -+ * @param uargs see _mali_uk_pp_and_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx, _mali_uk_pp_and_gp_start_job_s *uargs); -+ -+/** @brief Returns the number of Fragment Processors in the system -+ * -+ * @param args see _mali_uk_get_pp_number_of_cores_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args); -+ -+/** @brief Returns the version that all Fragment Processor cores are compatible with. -+ * -+ * This function may only be called when _mali_ukk_get_pp_number_of_cores() indicated at least one Fragment -+ * Processor core is available. -+ * -+ * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args); -+ -+/** @brief Disable Write-back unit(s) on specified job -+ * -+ * @param args see _mali_uk_get_pp_core_version_s in "mali_utgard_uk_types.h" -+ */ -+void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args); -+ -+ -+/** @} */ /* end group _mali_uk_pp */ -+ -+ -+/** @addtogroup _mali_uk_gp U/K Vertex Processor -+ * -+ * The Vertex Processor (aka GP (Geometry Processor)) functions provide the following functionality: -+ * - retrieving version of the Vertex Processors -+ * - determine number of Vertex Processors available -+ * - starting a job on a Vertex Processor -+ * -+ * @{ */ -+ -+/** @brief Issue a request to start a new job on a Vertex Processor. -+ * -+ * If the request fails args->status is set to _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE and you can -+ * try to start the job again. -+ * -+ * An existing job could be returned for requeueing if the new job has a higher priority than a previously started job -+ * which the hardware hasn't actually started processing yet. In this case the new job will be started and the -+ * existing one returned, otherwise the new job is started and the status field args->status is set to -+ * _MALI_UK_START_JOB_STARTED. -+ * -+ * Job completion can be awaited with _mali_ukk_wait_for_notification(). -+ * -+ * @param ctx user-kernel context (mali_session) -+ * @param uargs see _mali_uk_gp_start_job_s in "mali_utgard_uk_types.h". Use _mali_osk_copy_from_user to retrieve data! -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, _mali_uk_gp_start_job_s *uargs); -+ -+/** @brief Returns the number of Vertex Processors in the system. -+ * -+ * @param args see _mali_uk_get_gp_number_of_cores_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args); -+ -+/** @brief Returns the version that all Vertex Processor cores are compatible with. -+ * -+ * This function may only be called when _mali_uk_get_gp_number_of_cores() indicated at least one Vertex -+ * Processor core is available. -+ * -+ * @param args see _mali_uk_get_gp_core_version_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args); -+ -+/** @brief Resume or abort suspended Vertex Processor jobs. -+ * -+ * After receiving notification that a Vertex Processor job was suspended from -+ * _mali_ukk_wait_for_notification() you can use this function to resume or abort the job. -+ * -+ * @param args see _mali_uk_gp_suspend_response_s in "mali_utgard_uk_types.h" -+ * @return _MALI_OSK_ERR_OK on success, otherwise a suitable _mali_osk_errcode_t on failure. -+ */ -+_mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args); -+ -+/** @} */ /* end group _mali_uk_gp */ -+ -+#if defined(CONFIG_MALI400_PROFILING) -+/** @addtogroup _mali_uk_profiling U/K Timeline profiling module -+ * @{ */ -+ -+/** @brief Add event to profiling buffer. -+ * -+ * @param args see _mali_uk_profiling_add_event_s in "mali_utgard_uk_types.h" -+ */ -+_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args); -+ -+/** @brief Get profiling stream fd. -+ * -+ * @param args see _mali_uk_profiling_stream_fd_get_s in "mali_utgard_uk_types.h" -+ */ -+_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args); -+ -+/** @brief Profiling control set. -+ * -+ * @param args see _mali_uk_profiling_control_set_s in "mali_utgard_uk_types.h" -+ */ -+_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args); -+ -+/** @} */ /* end group _mali_uk_profiling */ -+#endif -+ -+/** @addtogroup _mali_uk_vsync U/K VSYNC reporting module -+ * @{ */ -+ -+/** @brief Report events related to vsync. -+ * -+ * @note Events should be reported when starting to wait for vsync and when the -+ * waiting is finished. This information can then be used in kernel space to -+ * complement the GPU utilization metric. -+ * -+ * @param args see _mali_uk_vsync_event_report_s in "mali_utgard_uk_types.h" -+ */ -+_mali_osk_errcode_t _mali_ukk_vsync_event_report(_mali_uk_vsync_event_report_s *args); -+ -+/** @} */ /* end group _mali_uk_vsync */ -+ -+/** @addtogroup _mali_sw_counters_report U/K Software counter reporting -+ * @{ */ -+ -+/** @brief Report software counters. -+ * -+ * @param args see _mali_uk_sw_counters_report_s in "mali_uk_types.h" -+ */ -+_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args); -+ -+/** @} */ /* end group _mali_sw_counters_report */ -+ -+/** @} */ /* end group u_k_api */ -+ -+/** @} */ /* end group uddapi */ -+ -+u32 _mali_ukk_report_memory_usage(void); -+ -+u32 _mali_ukk_report_total_memory_size(void); -+ -+u32 _mali_ukk_utilization_gp_pp(void); -+ -+u32 _mali_ukk_utilization_gp(void); -+ -+u32 _mali_ukk_utilization_pp(void); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_UKK_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c -new file mode 100755 -index 000000000..1911eff87 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.c -@@ -0,0 +1,147 @@ -+/** -+ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_ukk.h" -+#include "mali_uk_types.h" -+#include "mali_user_settings_db.h" -+#include "mali_session.h" -+ -+static u32 mali_user_settings[_MALI_UK_USER_SETTING_MAX]; -+const char *_mali_uk_user_setting_descriptions[] = _MALI_UK_USER_SETTING_DESCRIPTIONS; -+ -+static void mali_user_settings_notify(_mali_uk_user_setting_t setting, u32 value) -+{ -+ mali_bool done = MALI_FALSE; -+ -+ /* -+ * This function gets a bit complicated because we can't hold the session lock while -+ * allocating notification objects. -+ */ -+ -+ while (!done) { -+ u32 i; -+ u32 num_sessions_alloc; -+ u32 num_sessions_with_lock; -+ u32 used_notification_objects = 0; -+ _mali_osk_notification_t **notobjs; -+ -+ /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */ -+ num_sessions_alloc = mali_session_get_count(); -+ if (0 == num_sessions_alloc) { -+ /* No sessions to report to */ -+ return; -+ } -+ -+ notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc); -+ if (NULL == notobjs) { -+ MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n")); -+ return; -+ } -+ -+ for (i = 0; i < num_sessions_alloc; i++) { -+ notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_SETTINGS_CHANGED, -+ sizeof(_mali_uk_settings_changed_s)); -+ if (NULL != notobjs[i]) { -+ _mali_uk_settings_changed_s *data; -+ data = notobjs[i]->result_buffer; -+ -+ data->setting = setting; -+ data->value = value; -+ } else { -+ MALI_PRINT_ERROR(("Failed to notify user space session about setting change (alloc failure %u)\n", i)); -+ } -+ } -+ -+ mali_session_lock(); -+ -+ /* number of sessions will not change while we hold the lock */ -+ num_sessions_with_lock = mali_session_get_count(); -+ -+ if (num_sessions_alloc >= num_sessions_with_lock) { -+ /* We have allocated enough notification objects for all the sessions atm */ -+ struct mali_session_data *session, *tmp; -+ MALI_SESSION_FOREACH(session, tmp, link) { -+ MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc); -+ if (NULL != notobjs[used_notification_objects]) { -+ mali_session_send_notification(session, notobjs[used_notification_objects]); -+ notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */ -+ } -+ used_notification_objects++; -+ } -+ done = MALI_TRUE; -+ } -+ -+ mali_session_unlock(); -+ -+ /* Delete any remaining/unused notification objects */ -+ for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) { -+ if (NULL != notobjs[used_notification_objects]) { -+ _mali_osk_notification_delete(notobjs[used_notification_objects]); -+ } -+ } -+ -+ _mali_osk_free(notobjs); -+ } -+} -+ -+void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value) -+{ -+ mali_bool notify = MALI_FALSE; -+ -+ if (setting >= _MALI_UK_USER_SETTING_MAX) { -+ MALI_DEBUG_PRINT_ERROR(("Invalid user setting %ud\n")); -+ return; -+ } -+ -+ if (mali_user_settings[setting] != value) { -+ notify = MALI_TRUE; -+ } -+ -+ mali_user_settings[setting] = value; -+ -+ if (notify) { -+ mali_user_settings_notify(setting, value); -+ } -+} -+ -+u32 mali_get_user_setting(_mali_uk_user_setting_t setting) -+{ -+ if (setting >= _MALI_UK_USER_SETTING_MAX) { -+ return 0; -+ } -+ -+ return mali_user_settings[setting]; -+} -+ -+_mali_osk_errcode_t _mali_ukk_get_user_setting(_mali_uk_get_user_setting_s *args) -+{ -+ _mali_uk_user_setting_t setting; -+ MALI_DEBUG_ASSERT_POINTER(args); -+ -+ setting = args->setting; -+ -+ if (_MALI_UK_USER_SETTING_MAX > setting) { -+ args->value = mali_user_settings[setting]; -+ return _MALI_OSK_ERR_OK; -+ } else { -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+} -+ -+_mali_osk_errcode_t _mali_ukk_get_user_settings(_mali_uk_get_user_settings_s *args) -+{ -+ MALI_DEBUG_ASSERT_POINTER(args); -+ -+ _mali_osk_memcpy(args->settings, mali_user_settings, sizeof(mali_user_settings)); -+ -+ return _MALI_OSK_ERR_OK; -+} -diff --git a/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h -new file mode 100755 -index 000000000..da9c0630e ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/common/mali_user_settings_db.h -@@ -0,0 +1,39 @@ -+/** -+ * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_USER_SETTINGS_DB_H__ -+#define __MALI_USER_SETTINGS_DB_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include "mali_uk_types.h" -+ -+/** @brief Set Mali user setting in DB -+ * -+ * Update the DB with a new value for \a setting. If the value is different from theprevious set value running sessions will be notified of the change. -+ * -+ * @param setting the setting to be changed -+ * @param value the new value to set -+ */ -+void mali_set_user_setting(_mali_uk_user_setting_t setting, u32 value); -+ -+/** @brief Get current Mali user setting value from DB -+ * -+ * @param setting the setting to extract -+ * @return the value of the selected setting -+ */ -+u32 mali_get_user_setting(_mali_uk_user_setting_t setting); -+ -+#ifdef __cplusplus -+} -+#endif -+#endif /* __MALI_KERNEL_USER_SETTING__ */ -diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h -new file mode 100755 -index 000000000..7df55c951 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard.h -@@ -0,0 +1,526 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_utgard.h -+ * Defines types and interface exposed by the Mali Utgard device driver -+ */ -+ -+#ifndef __MALI_UTGARD_H__ -+#define __MALI_UTGARD_H__ -+ -+#include "mali_osk_types.h" -+#ifdef CONFIG_MALI_DEVFREQ -+#include -+#include "mali_pm_metrics.h" -+#ifdef CONFIG_DEVFREQ_THERMAL -+#include -+#endif -+#endif -+ -+#define MALI_GPU_NAME_UTGARD "mali-utgard" -+ -+ -+#define MALI_OFFSET_GP 0x00000 -+#define MALI_OFFSET_GP_MMU 0x03000 -+ -+#define MALI_OFFSET_PP0 0x08000 -+#define MALI_OFFSET_PP0_MMU 0x04000 -+#define MALI_OFFSET_PP1 0x0A000 -+#define MALI_OFFSET_PP1_MMU 0x05000 -+#define MALI_OFFSET_PP2 0x0C000 -+#define MALI_OFFSET_PP2_MMU 0x06000 -+#define MALI_OFFSET_PP3 0x0E000 -+#define MALI_OFFSET_PP3_MMU 0x07000 -+ -+#define MALI_OFFSET_PP4 0x28000 -+#define MALI_OFFSET_PP4_MMU 0x1C000 -+#define MALI_OFFSET_PP5 0x2A000 -+#define MALI_OFFSET_PP5_MMU 0x1D000 -+#define MALI_OFFSET_PP6 0x2C000 -+#define MALI_OFFSET_PP6_MMU 0x1E000 -+#define MALI_OFFSET_PP7 0x2E000 -+#define MALI_OFFSET_PP7_MMU 0x1F000 -+ -+#define MALI_OFFSET_L2_RESOURCE0 0x01000 -+#define MALI_OFFSET_L2_RESOURCE1 0x10000 -+#define MALI_OFFSET_L2_RESOURCE2 0x11000 -+ -+#define MALI400_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE0 -+#define MALI450_OFFSET_L2_CACHE0 MALI_OFFSET_L2_RESOURCE1 -+#define MALI450_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 -+#define MALI450_OFFSET_L2_CACHE2 MALI_OFFSET_L2_RESOURCE2 -+#define MALI470_OFFSET_L2_CACHE1 MALI_OFFSET_L2_RESOURCE0 -+ -+#define MALI_OFFSET_BCAST 0x13000 -+#define MALI_OFFSET_DLBU 0x14000 -+ -+#define MALI_OFFSET_PP_BCAST 0x16000 -+#define MALI_OFFSET_PP_BCAST_MMU 0x15000 -+ -+#define MALI_OFFSET_PMU 0x02000 -+#define MALI_OFFSET_DMA 0x12000 -+ -+/* Mali-300 */ -+ -+#define MALI_GPU_RESOURCES_MALI300(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ -+ MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) -+ -+#define MALI_GPU_RESOURCES_MALI300_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) \ -+ MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp_irq, pp_mmu_irq) -+ -+/* Mali-400 */ -+ -+#define MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCES_MALI400_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCES_MALI400_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCES_MALI400_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI400_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) -+ -+#define MALI_GPU_RESOURCES_MALI400_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCES_MALI400_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+ /* Mali-450 */ -+#define MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ -+ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) -+ -+#define MALI_GPU_RESOURCES_MALI450_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI450_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) -+ -+#define MALI_GPU_RESOURCES_MALI450_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI450_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ -+ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) -+ -+#define MALI_GPU_RESOURCES_MALI450_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI450_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP4, pp3_irq, base_addr + MALI_OFFSET_PP4_MMU, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP5, pp4_irq, base_addr + MALI_OFFSET_PP5_MMU, pp4_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP6, pp5_irq, base_addr + MALI_OFFSET_PP6_MMU, pp5_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ -+ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) -+ -+#define MALI_GPU_RESOURCES_MALI450_MP6_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI450_MP6(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE0) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI450_OFFSET_L2_CACHE2) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(4, base_addr + MALI_OFFSET_PP4, pp4_irq, base_addr + MALI_OFFSET_PP4_MMU, pp4_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(5, base_addr + MALI_OFFSET_PP5, pp5_irq, base_addr + MALI_OFFSET_PP5_MMU, pp5_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(6, base_addr + MALI_OFFSET_PP6, pp6_irq, base_addr + MALI_OFFSET_PP6_MMU, pp6_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(7, base_addr + MALI_OFFSET_PP7, pp7_irq, base_addr + MALI_OFFSET_PP7_MMU, pp7_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) \ -+ MALI_GPU_RESOURCE_DMA(base_addr + MALI_OFFSET_DMA) -+ -+#define MALI_GPU_RESOURCES_MALI450_MP8_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI450_MP8(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp4_irq, pp4_mmu_irq, pp5_irq, pp5_mmu_irq, pp6_irq, pp6_mmu_irq, pp7_irq, pp7_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+ /* Mali - 470 */ -+#define MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) -+ -+#define MALI_GPU_RESOURCES_MALI470_MP1_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI470_MP1(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) -+ -+#define MALI_GPU_RESOURCES_MALI470_MP2_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI470_MP2(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) -+ -+#define MALI_GPU_RESOURCES_MALI470_MP3_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI470_MP3(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_L2(base_addr + MALI470_OFFSET_L2_CACHE1) \ -+ MALI_GPU_RESOURCE_GP_WITH_MMU(base_addr + MALI_OFFSET_GP, gp_irq, base_addr + MALI_OFFSET_GP_MMU, gp_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(0, base_addr + MALI_OFFSET_PP0, pp0_irq, base_addr + MALI_OFFSET_PP0_MMU, pp0_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(1, base_addr + MALI_OFFSET_PP1, pp1_irq, base_addr + MALI_OFFSET_PP1_MMU, pp1_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(2, base_addr + MALI_OFFSET_PP2, pp2_irq, base_addr + MALI_OFFSET_PP2_MMU, pp2_mmu_irq) \ -+ MALI_GPU_RESOURCE_PP_WITH_MMU(3, base_addr + MALI_OFFSET_PP3, pp3_irq, base_addr + MALI_OFFSET_PP3_MMU, pp3_mmu_irq) \ -+ MALI_GPU_RESOURCE_BCAST(base_addr + MALI_OFFSET_BCAST) \ -+ MALI_GPU_RESOURCE_DLBU(base_addr + MALI_OFFSET_DLBU) \ -+ MALI_GPU_RESOURCE_PP_BCAST(base_addr + MALI_OFFSET_PP_BCAST, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PP_MMU_BCAST(base_addr + MALI_OFFSET_PP_BCAST_MMU) -+ -+#define MALI_GPU_RESOURCES_MALI470_MP4_PMU(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCES_MALI470_MP4(base_addr, gp_irq, gp_mmu_irq, pp0_irq, pp0_mmu_irq, pp1_irq, pp1_mmu_irq, pp2_irq, pp2_mmu_irq, pp3_irq, pp3_mmu_irq, pp_bcast_irq) \ -+ MALI_GPU_RESOURCE_PMU(base_addr + MALI_OFFSET_PMU) \ -+ -+#define MALI_GPU_RESOURCE_L2(addr) \ -+ { \ -+ .name = "Mali_L2", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = addr, \ -+ .end = addr + 0x200, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_GP(gp_addr, gp_irq) \ -+ { \ -+ .name = "Mali_GP", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = gp_addr, \ -+ .end = gp_addr + 0x100, \ -+ }, \ -+ { \ -+ .name = "Mali_GP_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = gp_irq, \ -+ .end = gp_irq, \ -+ }, \ -+ -+#define MALI_GPU_RESOURCE_GP_WITH_MMU(gp_addr, gp_irq, gp_mmu_addr, gp_mmu_irq) \ -+ { \ -+ .name = "Mali_GP", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = gp_addr, \ -+ .end = gp_addr + 0x100, \ -+ }, \ -+ { \ -+ .name = "Mali_GP_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = gp_irq, \ -+ .end = gp_irq, \ -+ }, \ -+ { \ -+ .name = "Mali_GP_MMU", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = gp_mmu_addr, \ -+ .end = gp_mmu_addr + 0x100, \ -+ }, \ -+ { \ -+ .name = "Mali_GP_MMU_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = gp_mmu_irq, \ -+ .end = gp_mmu_irq, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_PP(pp_addr, pp_irq) \ -+ { \ -+ .name = "Mali_PP", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = pp_addr, \ -+ .end = pp_addr + 0x1100, \ -+ }, \ -+ { \ -+ .name = "Mali_PP_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = pp_irq, \ -+ .end = pp_irq, \ -+ }, \ -+ -+#define MALI_GPU_RESOURCE_PP_WITH_MMU(id, pp_addr, pp_irq, pp_mmu_addr, pp_mmu_irq) \ -+ { \ -+ .name = "Mali_PP" #id, \ -+ .flags = IORESOURCE_MEM, \ -+ .start = pp_addr, \ -+ .end = pp_addr + 0x1100, \ -+ }, \ -+ { \ -+ .name = "Mali_PP" #id "_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = pp_irq, \ -+ .end = pp_irq, \ -+ }, \ -+ { \ -+ .name = "Mali_PP" #id "_MMU", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = pp_mmu_addr, \ -+ .end = pp_mmu_addr + 0x100, \ -+ }, \ -+ { \ -+ .name = "Mali_PP" #id "_MMU_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = pp_mmu_irq, \ -+ .end = pp_mmu_irq, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_MMU(mmu_addr, mmu_irq) \ -+ { \ -+ .name = "Mali_MMU", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = mmu_addr, \ -+ .end = mmu_addr + 0x100, \ -+ }, \ -+ { \ -+ .name = "Mali_MMU_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = mmu_irq, \ -+ .end = mmu_irq, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_PMU(pmu_addr) \ -+ { \ -+ .name = "Mali_PMU", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = pmu_addr, \ -+ .end = pmu_addr + 0x100, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_DMA(dma_addr) \ -+ { \ -+ .name = "Mali_DMA", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = dma_addr, \ -+ .end = dma_addr + 0x100, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_DLBU(dlbu_addr) \ -+ { \ -+ .name = "Mali_DLBU", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = dlbu_addr, \ -+ .end = dlbu_addr + 0x100, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_BCAST(bcast_addr) \ -+ { \ -+ .name = "Mali_Broadcast", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = bcast_addr, \ -+ .end = bcast_addr + 0x100, \ -+ }, -+ -+#define MALI_GPU_RESOURCE_PP_BCAST(pp_addr, pp_irq) \ -+ { \ -+ .name = "Mali_PP_Broadcast", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = pp_addr, \ -+ .end = pp_addr + 0x1100, \ -+ }, \ -+ { \ -+ .name = "Mali_PP_Broadcast_IRQ", \ -+ .flags = IORESOURCE_IRQ, \ -+ .start = pp_irq, \ -+ .end = pp_irq, \ -+ }, \ -+ -+#define MALI_GPU_RESOURCE_PP_MMU_BCAST(pp_mmu_bcast_addr) \ -+ { \ -+ .name = "Mali_PP_MMU_Broadcast", \ -+ .flags = IORESOURCE_MEM, \ -+ .start = pp_mmu_bcast_addr, \ -+ .end = pp_mmu_bcast_addr + 0x100, \ -+ }, -+ -+ struct mali_gpu_utilization_data { -+ unsigned int utilization_gpu; /* Utilization for GP and all PP cores combined, 0 = no utilization, 256 = full utilization */ -+ unsigned int utilization_gp; /* Utilization for GP core only, 0 = no utilization, 256 = full utilization */ -+ unsigned int utilization_pp; /* Utilization for all PP cores combined, 0 = no utilization, 256 = full utilization */ -+ }; -+ -+ struct mali_gpu_clk_item { -+ unsigned int clock; /* unit(MHz) */ -+ unsigned int vol; -+ }; -+ -+ struct mali_gpu_clock { -+ struct mali_gpu_clk_item *item; -+ unsigned int num_of_steps; -+ }; -+ -+ struct mali_gpu_device_data { -+ /* Shared GPU memory */ -+ unsigned long shared_mem_size; -+ -+ /* -+ * Mali PMU switch delay. -+ * Only needed if the power gates are connected to the PMU in a high fanout -+ * network. This value is the number of Mali clock cycles it takes to -+ * enable the power gates and turn on the power mesh. -+ * This value will have no effect if a daisy chain implementation is used. -+ */ -+ u32 pmu_switch_delay; -+ -+ /* Mali Dynamic power domain configuration in sequence from 0-11 -+ * GP PP0 PP1 PP2 PP3 PP4 PP5 PP6 PP7, L2$0 L2$1 L2$2 -+ */ -+ u16 pmu_domain_config[12]; -+ -+ /* Dedicated GPU memory range (physical). */ -+ unsigned long dedicated_mem_start; -+ unsigned long dedicated_mem_size; -+ -+ /* Frame buffer memory to be accessible by Mali GPU (physical) */ -+ unsigned long fb_start; -+ unsigned long fb_size; -+ -+ /* Max runtime [ms] for jobs */ -+ int max_job_runtime; -+ -+ /* Report GPU utilization and related control in this interval (specified in ms) */ -+ unsigned long control_interval; -+ -+ /* Function that will receive periodic GPU utilization numbers */ -+ void (*utilization_callback)(struct mali_gpu_utilization_data *data); -+ -+ /* Fuction that platform callback for freq setting, needed when CONFIG_MALI_DVFS enabled */ -+ int (*set_freq)(int setting_clock_step); -+ /* Function that platfrom report it's clock info which driver can set, needed when CONFIG_MALI_DVFS enabled */ -+ void (*get_clock_info)(struct mali_gpu_clock **data); -+ /* Function that get the current clock info, needed when CONFIG_MALI_DVFS enabled */ -+ int (*get_freq)(void); -+ /* Function that init the mali gpu secure mode */ -+ int (*secure_mode_init)(void); -+ /* Function that deinit the mali gpu secure mode */ -+ void (*secure_mode_deinit)(void); -+ /* Function that reset GPU and enable gpu secure mode */ -+ int (*gpu_reset_and_secure_mode_enable)(void); -+ /* Function that Reset GPU and disable gpu secure mode */ -+ int (*gpu_reset_and_secure_mode_disable)(void); -+ /* ipa related interface customer need register */ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ struct devfreq_cooling_power *gpu_cooling_ops; -+#endif -+ }; -+ -+ /** -+ * Pause the scheduling and power state changes of Mali device driver. -+ * mali_dev_resume() must always be called as soon as possible after this function -+ * in order to resume normal operation of the Mali driver. -+ */ -+ void mali_dev_pause(void); -+ -+ /** -+ * Resume scheduling and allow power changes in Mali device driver. -+ * This must always be called after mali_dev_pause(). -+ */ -+ void mali_dev_resume(void); -+ -+ /** @brief Set the desired number of PP cores to use. -+ * -+ * The internal Mali PMU will be used, if present, to physically power off the PP cores. -+ * -+ * @param num_cores The number of desired cores -+ * @return 0 on success, otherwise error. -EINVAL means an invalid number of cores was specified. -+ */ -+ int mali_perf_set_num_pp_cores(unsigned int num_cores); -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h -new file mode 100755 -index 000000000..686708eae ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_ioctl.h -@@ -0,0 +1,97 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_UTGARD_IOCTL_H__ -+#define __MALI_UTGARD_IOCTL_H__ -+ -+#include -+#include -+#include /* file system operations */ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/** -+ * @file mali_kernel_ioctl.h -+ * Interface to the Linux device driver. -+ * This file describes the interface needed to use the Linux device driver. -+ * Its interface is designed to used by the HAL implementation through a thin arch layer. -+ */ -+ -+/** -+ * ioctl commands -+ */ -+ -+#define MALI_IOC_BASE 0x82 -+#define MALI_IOC_CORE_BASE (_MALI_UK_CORE_SUBSYSTEM + MALI_IOC_BASE) -+#define MALI_IOC_MEMORY_BASE (_MALI_UK_MEMORY_SUBSYSTEM + MALI_IOC_BASE) -+#define MALI_IOC_PP_BASE (_MALI_UK_PP_SUBSYSTEM + MALI_IOC_BASE) -+#define MALI_IOC_GP_BASE (_MALI_UK_GP_SUBSYSTEM + MALI_IOC_BASE) -+#define MALI_IOC_PROFILING_BASE (_MALI_UK_PROFILING_SUBSYSTEM + MALI_IOC_BASE) -+#define MALI_IOC_VSYNC_BASE (_MALI_UK_VSYNC_SUBSYSTEM + MALI_IOC_BASE) -+ -+#define MALI_IOC_WAIT_FOR_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_WAIT_FOR_NOTIFICATION, _mali_uk_wait_for_notification_s) -+#define MALI_IOC_GET_API_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, u32) -+#define MALI_IOC_GET_API_VERSION_V2 _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_API_VERSION, _mali_uk_get_api_version_v2_s) -+/* rk_ext. */ -+#define MALI_IOC_GET_RK_KO_VERSION _IOWR(MALI_IOC_CORE_BASE, _MALI_GET_RK_KO_VERSION, _mali_rk_ko_version_s) -+#define MALI_IOC_POST_NOTIFICATION _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_POST_NOTIFICATION, _mali_uk_post_notification_s) -+#define MALI_IOC_GET_USER_SETTING _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTING, _mali_uk_get_user_setting_s) -+#define MALI_IOC_GET_USER_SETTINGS _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_GET_USER_SETTINGS, _mali_uk_get_user_settings_s) -+#define MALI_IOC_REQUEST_HIGH_PRIORITY _IOW (MALI_IOC_CORE_BASE, _MALI_UK_REQUEST_HIGH_PRIORITY, _mali_uk_request_high_priority_s) -+#define MALI_IOC_TIMELINE_GET_LATEST_POINT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_GET_LATEST_POINT, _mali_uk_timeline_get_latest_point_s) -+#define MALI_IOC_TIMELINE_WAIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_WAIT, _mali_uk_timeline_wait_s) -+#define MALI_IOC_TIMELINE_CREATE_SYNC_FENCE _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, _mali_uk_timeline_create_sync_fence_s) -+#define MALI_IOC_SOFT_JOB_START _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_START, _mali_uk_soft_job_start_s) -+#define MALI_IOC_SOFT_JOB_SIGNAL _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_SOFT_JOB_SIGNAL, _mali_uk_soft_job_signal_s) -+#define MALI_IOC_PENDING_SUBMIT _IOWR(MALI_IOC_CORE_BASE, _MALI_UK_PENDING_SUBMIT, _mali_uk_pending_submit_s) -+ -+#define MALI_IOC_MEM_ALLOC _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_ALLOC_MEM, _mali_uk_alloc_mem_s) -+#define MALI_IOC_MEM_FREE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_FREE_MEM, _mali_uk_free_mem_s) -+#define MALI_IOC_MEM_BIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_BIND_MEM, _mali_uk_bind_mem_s) -+#define MALI_IOC_MEM_UNBIND _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_UNBIND_MEM, _mali_uk_unbind_mem_s) -+#define MALI_IOC_MEM_COW _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MEM, _mali_uk_cow_mem_s) -+#define MALI_IOC_MEM_COW_MODIFY_RANGE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_COW_MODIFY_RANGE, _mali_uk_cow_modify_range_s) -+#define MALI_IOC_MEM_RESIZE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_RESIZE_MEM, _mali_uk_mem_resize_s) -+#define MALI_IOC_MEM_DMA_BUF_GET_SIZE _IOR(MALI_IOC_MEMORY_BASE, _MALI_UK_DMA_BUF_GET_SIZE, _mali_uk_dma_buf_get_size_s) -+#define MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE _IOR (MALI_IOC_MEMORY_BASE, _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, _mali_uk_query_mmu_page_table_dump_size_s) -+#define MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_DUMP_MMU_PAGE_TABLE, _mali_uk_dump_mmu_page_table_s) -+#define MALI_IOC_MEM_WRITE_SAFE _IOWR(MALI_IOC_MEMORY_BASE, _MALI_UK_MEM_WRITE_SAFE, _mali_uk_mem_write_safe_s) -+ -+#define MALI_IOC_PP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_START_JOB, _mali_uk_pp_start_job_s) -+#define MALI_IOC_PP_AND_GP_START_JOB _IOWR(MALI_IOC_PP_BASE, _MALI_UK_PP_AND_GP_START_JOB, _mali_uk_pp_and_gp_start_job_s) -+#define MALI_IOC_PP_NUMBER_OF_CORES_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_NUMBER_OF_CORES, _mali_uk_get_pp_number_of_cores_s) -+#define MALI_IOC_PP_CORE_VERSION_GET _IOR (MALI_IOC_PP_BASE, _MALI_UK_GET_PP_CORE_VERSION, _mali_uk_get_pp_core_version_s) -+#define MALI_IOC_PP_DISABLE_WB _IOW (MALI_IOC_PP_BASE, _MALI_UK_PP_DISABLE_WB, _mali_uk_pp_disable_wb_s) -+ -+#define MALI_IOC_GP2_START_JOB _IOWR(MALI_IOC_GP_BASE, _MALI_UK_GP_START_JOB, _mali_uk_gp_start_job_s) -+#define MALI_IOC_GP2_NUMBER_OF_CORES_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_NUMBER_OF_CORES, _mali_uk_get_gp_number_of_cores_s) -+#define MALI_IOC_GP2_CORE_VERSION_GET _IOR (MALI_IOC_GP_BASE, _MALI_UK_GET_GP_CORE_VERSION, _mali_uk_get_gp_core_version_s) -+#define MALI_IOC_GP2_SUSPEND_RESPONSE _IOW (MALI_IOC_GP_BASE, _MALI_UK_GP_SUSPEND_RESPONSE,_mali_uk_gp_suspend_response_s) -+ -+#define MALI_IOC_PROFILING_ADD_EVENT _IOWR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_ADD_EVENT, _mali_uk_profiling_add_event_s) -+#define MALI_IOC_PROFILING_REPORT_SW_COUNTERS _IOW (MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_REPORT_SW_COUNTERS, _mali_uk_sw_counters_report_s) -+#define MALI_IOC_PROFILING_MEMORY_USAGE_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_MEMORY_USAGE_GET, _mali_uk_profiling_memory_usage_get_s) -+#define MALI_IOC_PROFILING_STREAM_FD_GET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_STREAM_FD_GET, _mali_uk_profiling_stream_fd_get_s) -+#define MALI_IOC_PROILING_CONTROL_SET _IOR(MALI_IOC_PROFILING_BASE, _MALI_UK_PROFILING_CONTROL_SET, _mali_uk_profiling_control_set_s) -+ -+#define MALI_IOC_VSYNC_EVENT_REPORT _IOW (MALI_IOC_VSYNC_BASE, _MALI_UK_VSYNC_EVENT_REPORT, _mali_uk_vsync_event_report_s) -+ -+/* rk_ext : 对 r5p0 集æˆä¹‹åŽ, mali_so ä¸å†ä½¿ç”¨ä¸‹é¢çš„ ioctl, 而使用 MALI_IOC_GET_RK_KO_VERSION. */ -+#if 0 -+#define MALI_IOC_GET_MALI_VERSION_IN_RK30 _IOWR(MALI_IOC_CORE_BASE,_MALI_UK_GET_MALI_VERSION_IN_RK30,_mali_uk_get_mali_version_in_rk30_s *) -+#endif -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_UTGARD_IOCTL_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h -new file mode 100755 -index 000000000..17d31de93 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_events.h -@@ -0,0 +1,190 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef _MALI_UTGARD_PROFILING_EVENTS_H_ -+#define _MALI_UTGARD_PROFILING_EVENTS_H_ -+ -+/* -+ * The event ID is a 32 bit value consisting of different fields -+ * reserved, 4 bits, for future use -+ * event type, 4 bits, cinstr_profiling_event_type_t -+ * event channel, 8 bits, the source of the event. -+ * event data, 16 bit field, data depending on event type -+ */ -+ -+/** -+ * Specifies what kind of event this is -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_TYPE_SINGLE = 0 << 24, -+ MALI_PROFILING_EVENT_TYPE_START = 1 << 24, -+ MALI_PROFILING_EVENT_TYPE_STOP = 2 << 24, -+ MALI_PROFILING_EVENT_TYPE_SUSPEND = 3 << 24, -+ MALI_PROFILING_EVENT_TYPE_RESUME = 4 << 24, -+} cinstr_profiling_event_type_t; -+ -+ -+/** -+ * Secifies the channel/source of the event -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_CHANNEL_SOFTWARE = 0 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_GP0 = 1 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP0 = 5 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP1 = 6 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP2 = 7 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP3 = 8 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP4 = 9 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP5 = 10 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP6 = 11 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_PP7 = 12 << 16, -+ MALI_PROFILING_EVENT_CHANNEL_GPU = 21 << 16, -+} cinstr_profiling_event_channel_t; -+ -+ -+#define MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(num) (((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) + (num)) << 16) -+#define MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(num) (((MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) + (num)) << 16) -+ -+/** -+ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from software channel -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_NONE = 0, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_NEW_FRAME = 1, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_FLUSH = 2, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SWAP_BUFFERS = 3, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_FB_EVENT = 4, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE = 5, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE = 6, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_READBACK = 7, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_WRITEBACK = 8, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_ENTER_API_FUNC = 10, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC = 11, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_DISCARD_ATTACHMENTS = 13, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_TRY_LOCK = 53, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_LOCK = 54, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_UMP_UNLOCK = 55, -+ MALI_PROFILING_EVENT_REASON_SINGLE_LOCK_CONTENDED = 56, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_MALI_FENCE_DUP = 57, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_SET_PP_JOB_FENCE = 58, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_WAIT_SYNC = 59, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_FENCE_SYNC = 60, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_CREATE_NATIVE_FENCE_SYNC = 61, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FENCE_FLUSH = 62, -+ MALI_PROFILING_EVENT_REASON_SINGLE_SW_EGL_FLUSH_SERVER_WAITS = 63, -+} cinstr_profiling_event_reason_single_sw_t; -+ -+/** -+ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel -+ * to inform whether the core is physical or virtual -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL = 0, -+ MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL = 1, -+} cinstr_profiling_event_reason_start_stop_hw_t; -+ -+/** -+ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_START/STOP is used from software channel -+ */ -+typedef enum { -+ /*MALI_PROFILING_EVENT_REASON_START_STOP_SW_NONE = 0,*/ -+ MALI_PROFILING_EVENT_REASON_START_STOP_SW_MALI = 1, -+ MALI_PROFILING_EVENT_REASON_START_STOP_SW_CALLBACK_THREAD = 2, -+ MALI_PROFILING_EVENT_REASON_START_STOP_SW_WORKER_THREAD = 3, -+ MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF = 4, -+ MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF = 5, -+} cinstr_profiling_event_reason_start_stop_sw_t; -+ -+/** -+ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SUSPEND/RESUME is used from software channel -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_NONE = 0, /* used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PIPELINE_FULL = 1, /* NOT used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VSYNC = 26, /* used in some build configurations */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_WAIT = 27, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_IFRAME_SYNC = 28, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_FILTER_CLEANUP = 29, /* used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_VG_WAIT_TEXTURE = 30, /* used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_MIPLEVEL = 31, /* used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GLES_WAIT_READPIXELS = 32, /* used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SWAP_IMMEDIATE = 33, /* NOT used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_QUEUE_BUFFER = 34, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_ICS_DEQUEUE_BUFFER = 35, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_UMP_LOCK = 36, /* Not currently used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_GLOBAL_LOCK = 37, /* Not currently used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_X11_SWAP = 38, /* Not currently used */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_MALI_EGL_IMAGE_SYNC_WAIT = 39, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_GP_JOB_HANDLING = 40, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_PP_JOB_HANDLING = 41, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_MERGE = 42, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_MALI_FENCE_DUP = 43, -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_FLUSH_SERVER_WAITS = 44, -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_EGL_WAIT_SYNC = 45, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_JOBS_WAIT = 46, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOFRAMES_WAIT = 47, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_FB_NOJOBS_WAIT = 48, /* USED */ -+ MALI_PROFILING_EVENT_REASON_SUSPEND_RESUME_SW_SUBMIT_LIMITER_WAIT = 49, /* USED */ -+} cinstr_profiling_event_reason_suspend_resume_sw_t; -+ -+/** -+ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from a HW channel (GPx+PPx) -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_REASON_SINGLE_HW_NONE = 0, -+ MALI_PROFILING_EVENT_REASON_SINGLE_HW_INTERRUPT = 1, -+ MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH = 2, -+} cinstr_profiling_event_reason_single_hw_t; -+ -+/** -+ * These events are applicable when the type MALI_PROFILING_EVENT_TYPE_SINGLE is used from the GPU channel -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_NONE = 0, -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE = 1, -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS = 2, -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS = 3, -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS = 4, -+} cinstr_profiling_event_reason_single_gpu_t; -+ -+/** -+ * These values are applicable for the 3rd data parameter when -+ * the type MALI_PROFILING_EVENT_TYPE_START is used from the software channel -+ * with the MALI_PROFILING_EVENT_REASON_START_STOP_BOTTOM_HALF reason. -+ */ -+typedef enum { -+ MALI_PROFILING_EVENT_DATA_CORE_GP0 = 1, -+ MALI_PROFILING_EVENT_DATA_CORE_PP0 = 5, -+ MALI_PROFILING_EVENT_DATA_CORE_PP1 = 6, -+ MALI_PROFILING_EVENT_DATA_CORE_PP2 = 7, -+ MALI_PROFILING_EVENT_DATA_CORE_PP3 = 8, -+ MALI_PROFILING_EVENT_DATA_CORE_PP4 = 9, -+ MALI_PROFILING_EVENT_DATA_CORE_PP5 = 10, -+ MALI_PROFILING_EVENT_DATA_CORE_PP6 = 11, -+ MALI_PROFILING_EVENT_DATA_CORE_PP7 = 12, -+ MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU = 22, /* GP0 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU = 26, /* PP0 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP1_MMU = 27, /* PP1 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP2_MMU = 28, /* PP2 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP3_MMU = 29, /* PP3 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP4_MMU = 30, /* PP4 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP5_MMU = 31, /* PP5 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP6_MMU = 32, /* PP6 + 21 */ -+ MALI_PROFILING_EVENT_DATA_CORE_PP7_MMU = 33, /* PP7 + 21 */ -+ -+} cinstr_profiling_event_data_core_t; -+ -+#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0 + (num)) -+#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_GP0_MMU + (num)) -+#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0 + (num)) -+#define MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(num) (MALI_PROFILING_EVENT_DATA_CORE_PP0_MMU + (num)) -+ -+ -+#endif /*_MALI_UTGARD_PROFILING_EVENTS_H_*/ -diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h -new file mode 100755 -index 000000000..c1927d145 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_profiling_gator_api.h -@@ -0,0 +1,305 @@ -+/* -+ * Copyright (C) 2013, 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_UTGARD_PROFILING_GATOR_API_H__ -+#define __MALI_UTGARD_PROFILING_GATOR_API_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#define MALI_PROFILING_API_VERSION 4 -+ -+#define MAX_NUM_L2_CACHE_CORES 3 -+#define MAX_NUM_FP_CORES 8 -+#define MAX_NUM_VP_CORES 1 -+ -+#define _MALI_SPCIAL_COUNTER_DESCRIPTIONS \ -+ { \ -+ "Filmstrip_cnt0", \ -+ "Frequency", \ -+ "Voltage", \ -+ "vertex", \ -+ "fragment", \ -+ "Total_alloc_pages", \ -+ }; -+ -+#define _MALI_MEM_COUTNER_DESCRIPTIONS \ -+ { \ -+ "untyped_memory", \ -+ "vertex_index_buffer", \ -+ "texture_buffer", \ -+ "varying_buffer", \ -+ "render_target", \ -+ "pbuffer_buffer", \ -+ "plbu_heap", \ -+ "pointer_array_buffer", \ -+ "slave_tilelist", \ -+ "untyped_gp_cmdlist", \ -+ "polygon_cmdlist", \ -+ "texture_descriptor", \ -+ "render_state_word", \ -+ "shader", \ -+ "stream_buffer", \ -+ "fragment_stack", \ -+ "uniform", \ -+ "untyped_frame_pool", \ -+ "untyped_surface", \ -+ }; -+ -+/** The list of events supported by the Mali DDK. */ -+typedef enum { -+ /* Vertex processor activity */ -+ ACTIVITY_VP_0 = 0, -+ -+ /* Fragment processor activity */ -+ ACTIVITY_FP_0, -+ ACTIVITY_FP_1, -+ ACTIVITY_FP_2, -+ ACTIVITY_FP_3, -+ ACTIVITY_FP_4, -+ ACTIVITY_FP_5, -+ ACTIVITY_FP_6, -+ ACTIVITY_FP_7, -+ -+ /* L2 cache counters */ -+ COUNTER_L2_0_C0, -+ COUNTER_L2_0_C1, -+ COUNTER_L2_1_C0, -+ COUNTER_L2_1_C1, -+ COUNTER_L2_2_C0, -+ COUNTER_L2_2_C1, -+ -+ /* Vertex processor counters */ -+ COUNTER_VP_0_C0, -+ COUNTER_VP_0_C1, -+ -+ /* Fragment processor counters */ -+ COUNTER_FP_0_C0, -+ COUNTER_FP_0_C1, -+ COUNTER_FP_1_C0, -+ COUNTER_FP_1_C1, -+ COUNTER_FP_2_C0, -+ COUNTER_FP_2_C1, -+ COUNTER_FP_3_C0, -+ COUNTER_FP_3_C1, -+ COUNTER_FP_4_C0, -+ COUNTER_FP_4_C1, -+ COUNTER_FP_5_C0, -+ COUNTER_FP_5_C1, -+ COUNTER_FP_6_C0, -+ COUNTER_FP_6_C1, -+ COUNTER_FP_7_C0, -+ COUNTER_FP_7_C1, -+ -+ /* -+ * If more hardware counters are added, the _mali_osk_hw_counter_table -+ * below should also be updated. -+ */ -+ -+ /* EGL software counters */ -+ COUNTER_EGL_BLIT_TIME, -+ -+ /* GLES software counters */ -+ COUNTER_GLES_DRAW_ELEMENTS_CALLS, -+ COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, -+ COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, -+ COUNTER_GLES_DRAW_ARRAYS_CALLS, -+ COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, -+ COUNTER_GLES_DRAW_POINTS, -+ COUNTER_GLES_DRAW_LINES, -+ COUNTER_GLES_DRAW_LINE_LOOP, -+ COUNTER_GLES_DRAW_LINE_STRIP, -+ COUNTER_GLES_DRAW_TRIANGLES, -+ COUNTER_GLES_DRAW_TRIANGLE_STRIP, -+ COUNTER_GLES_DRAW_TRIANGLE_FAN, -+ COUNTER_GLES_NON_VBO_DATA_COPY_TIME, -+ COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, -+ COUNTER_GLES_UPLOAD_TEXTURE_TIME, -+ COUNTER_GLES_UPLOAD_VBO_TIME, -+ COUNTER_GLES_NUM_FLUSHES, -+ COUNTER_GLES_NUM_VSHADERS_GENERATED, -+ COUNTER_GLES_NUM_FSHADERS_GENERATED, -+ COUNTER_GLES_VSHADER_GEN_TIME, -+ COUNTER_GLES_FSHADER_GEN_TIME, -+ COUNTER_GLES_INPUT_TRIANGLES, -+ COUNTER_GLES_VXCACHE_HIT, -+ COUNTER_GLES_VXCACHE_MISS, -+ COUNTER_GLES_VXCACHE_COLLISION, -+ COUNTER_GLES_CULLED_TRIANGLES, -+ COUNTER_GLES_CULLED_LINES, -+ COUNTER_GLES_BACKFACE_TRIANGLES, -+ COUNTER_GLES_GBCLIP_TRIANGLES, -+ COUNTER_GLES_GBCLIP_LINES, -+ COUNTER_GLES_TRIANGLES_DRAWN, -+ COUNTER_GLES_DRAWCALL_TIME, -+ COUNTER_GLES_TRIANGLES_COUNT, -+ COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, -+ COUNTER_GLES_STRIP_TRIANGLES_COUNT, -+ COUNTER_GLES_FAN_TRIANGLES_COUNT, -+ COUNTER_GLES_LINES_COUNT, -+ COUNTER_GLES_INDEPENDENT_LINES_COUNT, -+ COUNTER_GLES_STRIP_LINES_COUNT, -+ COUNTER_GLES_LOOP_LINES_COUNT, -+ -+ /* Special counter */ -+ -+ /* Framebuffer capture pseudo-counter */ -+ COUNTER_FILMSTRIP, -+ COUNTER_FREQUENCY, -+ COUNTER_VOLTAGE, -+ COUNTER_VP_ACTIVITY, -+ COUNTER_FP_ACTIVITY, -+ COUNTER_TOTAL_ALLOC_PAGES, -+ -+ /* Memory usage counter */ -+ COUNTER_MEM_UNTYPED, -+ COUNTER_MEM_VB_IB, -+ COUNTER_MEM_TEXTURE, -+ COUNTER_MEM_VARYING, -+ COUNTER_MEM_RT, -+ COUNTER_MEM_PBUFFER, -+ /* memory usages for gp command */ -+ COUNTER_MEM_PLBU_HEAP, -+ COUNTER_MEM_POINTER_ARRAY, -+ COUNTER_MEM_SLAVE_TILELIST, -+ COUNTER_MEM_UNTYPE_GP_CMDLIST, -+ /* memory usages for polygon list command */ -+ COUNTER_MEM_POLYGON_CMDLIST, -+ /* memory usages for pp command */ -+ COUNTER_MEM_TD, -+ COUNTER_MEM_RSW, -+ /* other memory usages */ -+ COUNTER_MEM_SHADER, -+ COUNTER_MEM_STREAMS, -+ COUNTER_MEM_FRAGMENT_STACK, -+ COUNTER_MEM_UNIFORM, -+ /* Special mem usage, which is used for mem pool allocation */ -+ COUNTER_MEM_UNTYPE_MEM_POOL, -+ COUNTER_MEM_UNTYPE_SURFACE, -+ -+ NUMBER_OF_EVENTS -+} _mali_osk_counter_id; -+ -+#define FIRST_ACTIVITY_EVENT ACTIVITY_VP_0 -+#define LAST_ACTIVITY_EVENT ACTIVITY_FP_7 -+ -+#define FIRST_HW_COUNTER COUNTER_L2_0_C0 -+#define LAST_HW_COUNTER COUNTER_FP_7_C1 -+ -+#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME -+#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT -+ -+#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP -+#define LAST_SPECIAL_COUNTER COUNTER_TOTAL_ALLOC_PAGES -+ -+#define FIRST_MEM_COUNTER COUNTER_MEM_UNTYPED -+#define LAST_MEM_COUNTER COUNTER_MEM_UNTYPE_SURFACE -+ -+#define MALI_PROFILING_MEM_COUNTERS_NUM (LAST_MEM_COUNTER - FIRST_MEM_COUNTER + 1) -+#define MALI_PROFILING_SPECIAL_COUNTERS_NUM (LAST_SPECIAL_COUNTER - FIRST_SPECIAL_COUNTER + 1) -+#define MALI_PROFILING_SW_COUNTERS_NUM (LAST_SW_COUNTER - FIRST_SW_COUNTER + 1) -+ -+/** -+ * Define the stream header type for porfiling stream. -+ */ -+#define STREAM_HEADER_FRAMEBUFFER 0x05 /* The stream packet header type for framebuffer dumping. */ -+#define STREAM_HEADER_COUNTER_VALUE 0x09 /* The stream packet header type for hw/sw/memory counter sampling. */ -+#define STREAM_HEADER_CORE_ACTIVITY 0x0a /* The stream packet header type for activity counter sampling. */ -+#define STREAM_HEADER_SIZE 5 -+ -+/** -+ * Define the packet header type of profiling control packet. -+ */ -+#define PACKET_HEADER_ERROR 0x80 /* The response packet header type if error. */ -+#define PACKET_HEADER_ACK 0x81 /* The response packet header type if OK. */ -+#define PACKET_HEADER_COUNTERS_REQUEST 0x82 /* The control packet header type to request counter information from ddk. */ -+#define PACKET_HEADER_COUNTERS_ACK 0x83 /* The response packet header type to send out counter information. */ -+#define PACKET_HEADER_COUNTERS_ENABLE 0x84 /* The control packet header type to enable counters. */ -+#define PACKET_HEADER_START_CAPTURE_VALUE 0x85 /* The control packet header type to start capture values. */ -+ -+#define PACKET_HEADER_SIZE 5 -+ -+/** -+ * Structure to pass performance counter data of a Mali core -+ */ -+typedef struct _mali_profiling_core_counters { -+ u32 source0; -+ u32 value0; -+ u32 source1; -+ u32 value1; -+} _mali_profiling_core_counters; -+ -+/** -+ * Structure to pass performance counter data of Mali L2 cache cores -+ */ -+typedef struct _mali_profiling_l2_counter_values { -+ struct _mali_profiling_core_counters cores[MAX_NUM_L2_CACHE_CORES]; -+} _mali_profiling_l2_counter_values; -+ -+/** -+ * Structure to pass data defining Mali instance in use: -+ * -+ * mali_product_id - Mali product id -+ * mali_version_major - Mali version major number -+ * mali_version_minor - Mali version minor number -+ * num_of_l2_cores - number of L2 cache cores -+ * num_of_fp_cores - number of fragment processor cores -+ * num_of_vp_cores - number of vertex processor cores -+ */ -+typedef struct _mali_profiling_mali_version { -+ u32 mali_product_id; -+ u32 mali_version_major; -+ u32 mali_version_minor; -+ u32 num_of_l2_cores; -+ u32 num_of_fp_cores; -+ u32 num_of_vp_cores; -+} _mali_profiling_mali_version; -+ -+/** -+ * Structure to define the mali profiling counter struct. -+ */ -+typedef struct mali_profiling_counter { -+ char counter_name[40]; -+ u32 counter_id; -+ u32 counter_event; -+ u32 prev_counter_value; -+ u32 current_counter_value; -+ u32 key; -+ int enabled; -+} mali_profiling_counter; -+ -+/* -+ * List of possible actions to be controlled by Streamline. -+ * The following numbers are used by gator to control the frame buffer dumping and s/w counter reporting. -+ * We cannot use the enums in mali_uk_types.h because they are unknown inside gator. -+ */ -+#define FBDUMP_CONTROL_ENABLE (1) -+#define FBDUMP_CONTROL_RATE (2) -+#define SW_COUNTER_ENABLE (3) -+#define FBDUMP_CONTROL_RESIZE_FACTOR (4) -+#define MEM_COUNTER_ENABLE (5) -+#define ANNOTATE_PROFILING_ENABLE (6) -+ -+void _mali_profiling_control(u32 action, u32 value); -+ -+u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values); -+ -+int _mali_profiling_set_event(u32 counter_id, s32 event_id); -+ -+u32 _mali_profiling_get_api_version(void); -+ -+void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_UTGARD_PROFILING_GATOR_API_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h -new file mode 100755 -index 000000000..34656f09b ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/include/linux/mali/mali_utgard_uk_types.h -@@ -0,0 +1,1108 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_uk_types.h -+ * Defines the types and constants used in the user-kernel interface -+ */ -+ -+#ifndef __MALI_UTGARD_UK_TYPES_H__ -+#define __MALI_UTGARD_UK_TYPES_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/* Iteration functions depend on these values being consecutive. */ -+#define MALI_UK_TIMELINE_GP 0 -+#define MALI_UK_TIMELINE_PP 1 -+#define MALI_UK_TIMELINE_SOFT 2 -+#define MALI_UK_TIMELINE_MAX 3 -+ -+#define MALI_UK_BIG_VARYING_SIZE (1024*1024*2) -+ -+typedef struct { -+ u32 points[MALI_UK_TIMELINE_MAX]; -+ s32 sync_fd; -+} _mali_uk_fence_t; -+ -+/** -+ * @addtogroup uddapi Unified Device Driver (UDD) APIs -+ * -+ * @{ -+ */ -+ -+/** -+ * @addtogroup u_k_api UDD User/Kernel Interface (U/K) APIs -+ * -+ * @{ -+ */ -+ -+/** @defgroup _mali_uk_core U/K Core -+ * @{ */ -+ -+/** Definition of subsystem numbers, to assist in creating a unique identifier -+ * for each U/K call. -+ * -+ * @see _mali_uk_functions */ -+typedef enum { -+ _MALI_UK_CORE_SUBSYSTEM, /**< Core Group of U/K calls */ -+ _MALI_UK_MEMORY_SUBSYSTEM, /**< Memory Group of U/K calls */ -+ _MALI_UK_PP_SUBSYSTEM, /**< Fragment Processor Group of U/K calls */ -+ _MALI_UK_GP_SUBSYSTEM, /**< Vertex Processor Group of U/K calls */ -+ _MALI_UK_PROFILING_SUBSYSTEM, /**< Profiling Group of U/K calls */ -+ _MALI_UK_VSYNC_SUBSYSTEM, /**< VSYNC Group of U/K calls */ -+} _mali_uk_subsystem_t; -+ -+/** Within a function group each function has its unique sequence number -+ * to assist in creating a unique identifier for each U/K call. -+ * -+ * An ordered pair of numbers selected from -+ * ( \ref _mali_uk_subsystem_t,\ref _mali_uk_functions) will uniquely identify the -+ * U/K call across all groups of functions, and all functions. */ -+typedef enum { -+ /** Core functions */ -+ -+ _MALI_UK_OPEN = 0, /**< _mali_ukk_open() */ -+ _MALI_UK_CLOSE, /**< _mali_ukk_close() */ -+ _MALI_UK_WAIT_FOR_NOTIFICATION, /**< _mali_ukk_wait_for_notification() */ -+ _MALI_UK_GET_API_VERSION, /**< _mali_ukk_get_api_version() */ -+ _MALI_UK_POST_NOTIFICATION, /**< _mali_ukk_post_notification() */ -+ _MALI_UK_GET_USER_SETTING, /**< _mali_ukk_get_user_setting() *//**< [out] */ -+ _MALI_UK_GET_USER_SETTINGS, /**< _mali_ukk_get_user_settings() *//**< [out] */ -+ _MALI_UK_REQUEST_HIGH_PRIORITY, /**< _mali_ukk_request_high_priority() */ -+ _MALI_UK_TIMELINE_GET_LATEST_POINT, /**< _mali_ukk_timeline_get_latest_point() */ -+ _MALI_UK_TIMELINE_WAIT, /**< _mali_ukk_timeline_wait() */ -+ _MALI_UK_TIMELINE_CREATE_SYNC_FENCE, /**< _mali_ukk_timeline_create_sync_fence() */ -+ _MALI_UK_SOFT_JOB_START, /**< _mali_ukk_soft_job_start() */ -+ _MALI_UK_SOFT_JOB_SIGNAL, /**< _mali_ukk_soft_job_signal() */ -+ _MALI_UK_PENDING_SUBMIT, /**< _mali_ukk_pending_submit() */ -+ -+ _MALI_GET_RK_KO_VERSION, /* rk_ext */ -+ _MALI_UK_GET_MALI_VERSION_IN_RK30, -+ -+ /** Memory functions */ -+ -+ _MALI_UK_ALLOC_MEM = 0, /**< _mali_ukk_alloc_mem() */ -+ _MALI_UK_FREE_MEM, /**< _mali_ukk_free_mem() */ -+ _MALI_UK_BIND_MEM, /**< _mali_ukk_mem_bind() */ -+ _MALI_UK_UNBIND_MEM, /**< _mali_ukk_mem_unbind() */ -+ _MALI_UK_COW_MEM, /**< _mali_ukk_mem_cow() */ -+ _MALI_UK_COW_MODIFY_RANGE, /**< _mali_ukk_mem_cow_modify_range() */ -+ _MALI_UK_RESIZE_MEM, /**<._mali_ukk_mem_resize() */ -+ _MALI_UK_QUERY_MMU_PAGE_TABLE_DUMP_SIZE, /**< _mali_ukk_mem_get_mmu_page_table_dump_size() */ -+ _MALI_UK_DUMP_MMU_PAGE_TABLE, /**< _mali_ukk_mem_dump_mmu_page_table() */ -+ _MALI_UK_DMA_BUF_GET_SIZE, /**< _mali_ukk_dma_buf_get_size() */ -+ _MALI_UK_MEM_WRITE_SAFE, /**< _mali_uku_mem_write_safe() */ -+ -+ /** Common functions for each core */ -+ -+ _MALI_UK_START_JOB = 0, /**< Start a Fragment/Vertex Processor Job on a core */ -+ _MALI_UK_GET_NUMBER_OF_CORES, /**< Get the number of Fragment/Vertex Processor cores */ -+ _MALI_UK_GET_CORE_VERSION, /**< Get the Fragment/Vertex Processor version compatible with all cores */ -+ -+ /** Fragment Processor Functions */ -+ -+ _MALI_UK_PP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_pp_start_job() */ -+ _MALI_UK_GET_PP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_pp_number_of_cores() */ -+ _MALI_UK_GET_PP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_pp_core_version() */ -+ _MALI_UK_PP_DISABLE_WB, /**< _mali_ukk_pp_job_disable_wb() */ -+ _MALI_UK_PP_AND_GP_START_JOB, /**< _mali_ukk_pp_and_gp_start_job() */ -+ -+ /** Vertex Processor Functions */ -+ -+ _MALI_UK_GP_START_JOB = _MALI_UK_START_JOB, /**< _mali_ukk_gp_start_job() */ -+ _MALI_UK_GET_GP_NUMBER_OF_CORES = _MALI_UK_GET_NUMBER_OF_CORES, /**< _mali_ukk_get_gp_number_of_cores() */ -+ _MALI_UK_GET_GP_CORE_VERSION = _MALI_UK_GET_CORE_VERSION, /**< _mali_ukk_get_gp_core_version() */ -+ _MALI_UK_GP_SUSPEND_RESPONSE, /**< _mali_ukk_gp_suspend_response() */ -+ -+ /** Profiling functions */ -+ -+ _MALI_UK_PROFILING_ADD_EVENT = 0, /**< __mali_uku_profiling_add_event() */ -+ _MALI_UK_PROFILING_REPORT_SW_COUNTERS,/**< __mali_uku_profiling_report_sw_counters() */ -+ _MALI_UK_PROFILING_MEMORY_USAGE_GET, /**< __mali_uku_profiling_memory_usage_get() */ -+ _MALI_UK_PROFILING_STREAM_FD_GET, /** < __mali_uku_profiling_stream_fd_get() */ -+ _MALI_UK_PROFILING_CONTROL_SET, /** < __mali_uku_profiling_control_set() */ -+ -+ /** VSYNC reporting fuctions */ -+ _MALI_UK_VSYNC_EVENT_REPORT = 0, /**< _mali_ukk_vsync_event_report() */ -+} _mali_uk_functions; -+ -+/** @defgroup _mali_uk_getsysteminfo U/K Get System Info -+ * @{ */ -+ -+/** -+ * Type definition for the core version number. -+ * Used when returning the version number read from a core -+ * -+ * Its format is that of the 32-bit Version register for a particular core. -+ * Refer to the "Mali200 and MaliGP2 3D Graphics Processor Technical Reference -+ * Manual", ARM DDI 0415C, for more information. -+ */ -+typedef u32 _mali_core_version; -+ -+/** @} */ /* end group _mali_uk_core */ -+ -+ -+/** @defgroup _mali_uk_gp U/K Vertex Processor -+ * @{ */ -+ -+/** @defgroup _mali_uk_gp_suspend_response_s Vertex Processor Suspend Response -+ * @{ */ -+ -+/** @brief Arguments for _mali_ukk_gp_suspend_response() -+ * -+ * When _mali_wait_for_notification() receives notification that a -+ * Vertex Processor job was suspended, you need to send a response to indicate -+ * what needs to happen with this job. You can either abort or resume the job. -+ * -+ * - set @c code to indicate response code. This is either @c _MALIGP_JOB_ABORT or -+ * @c _MALIGP_JOB_RESUME_WITH_NEW_HEAP to indicate you will provide a new heap -+ * for the job that will resolve the out of memory condition for the job. -+ * - copy the @c cookie value from the @c _mali_uk_gp_job_suspended_s notification; -+ * this is an identifier for the suspended job -+ * - set @c arguments[0] and @c arguments[1] to zero if you abort the job. If -+ * you resume it, @c argument[0] should specify the Mali start address for the new -+ * heap and @c argument[1] the Mali end address of the heap. -+ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() -+ * -+ */ -+typedef enum _maligp_job_suspended_response_code { -+ _MALIGP_JOB_ABORT, /**< Abort the Vertex Processor job */ -+ _MALIGP_JOB_RESUME_WITH_NEW_HEAP /**< Resume the Vertex Processor job with a new heap */ -+} _maligp_job_suspended_response_code; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 cookie; /**< [in] cookie from the _mali_uk_gp_job_suspended_s notification */ -+ _maligp_job_suspended_response_code code; /**< [in] abort or resume response code, see \ref _maligp_job_suspended_response_code */ -+ u32 arguments[2]; /**< [in] 0 when aborting a job. When resuming a job, the Mali start and end address for a new heap to resume the job with */ -+} _mali_uk_gp_suspend_response_s; -+ -+/** @} */ /* end group _mali_uk_gp_suspend_response_s */ -+ -+/** @defgroup _mali_uk_gpstartjob_s Vertex Processor Start Job -+ * @{ */ -+ -+/** @brief Status indicating the result of the execution of a Vertex or Fragment processor job */ -+typedef enum { -+ _MALI_UK_JOB_STATUS_END_SUCCESS = 1 << (16 + 0), -+ _MALI_UK_JOB_STATUS_END_OOM = 1 << (16 + 1), -+ _MALI_UK_JOB_STATUS_END_ABORT = 1 << (16 + 2), -+ _MALI_UK_JOB_STATUS_END_TIMEOUT_SW = 1 << (16 + 3), -+ _MALI_UK_JOB_STATUS_END_HANG = 1 << (16 + 4), -+ _MALI_UK_JOB_STATUS_END_SEG_FAULT = 1 << (16 + 5), -+ _MALI_UK_JOB_STATUS_END_ILLEGAL_JOB = 1 << (16 + 6), -+ _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR = 1 << (16 + 7), -+ _MALI_UK_JOB_STATUS_END_SHUTDOWN = 1 << (16 + 8), -+ _MALI_UK_JOB_STATUS_END_SYSTEM_UNUSABLE = 1 << (16 + 9) -+} _mali_uk_job_status; -+ -+#define MALIGP2_NUM_REGS_FRAME (6) -+ -+/** @brief Arguments for _mali_ukk_gp_start_job() -+ * -+ * To start a Vertex Processor job -+ * - associate the request with a reference to a @c mali_gp_job_info by setting -+ * user_job_ptr to the address of the @c mali_gp_job_info of the job. -+ * - set @c priority to the priority of the @c mali_gp_job_info -+ * - specify a timeout for the job by setting @c watchdog_msecs to the number of -+ * milliseconds the job is allowed to run. Specifying a value of 0 selects the -+ * default timeout in use by the device driver. -+ * - copy the frame registers from the @c mali_gp_job_info into @c frame_registers. -+ * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero -+ * for a non-instrumented build. For an instrumented build you can use up -+ * to two performance counters. Set the corresponding bit in @c perf_counter_flag -+ * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify -+ * the source of what needs to get counted (e.g. number of vertex loader -+ * cache hits). For source id values, see ARM DDI0415A, Table 3-60. -+ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() -+ * -+ * When @c _mali_ukk_gp_start_job() returns @c _MALI_OSK_ERR_OK, status contains the -+ * result of the request (see \ref _mali_uk_start_job_status). If the job could -+ * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be -+ * tried again. -+ * -+ * After the job has started, @c _mali_wait_for_notification() will be notified -+ * that the job finished or got suspended. It may get suspended due to -+ * resource shortage. If it finished (see _mali_ukk_wait_for_notification()) -+ * the notification will contain a @c _mali_uk_gp_job_finished_s result. If -+ * it got suspended the notification will contain a @c _mali_uk_gp_job_suspended_s -+ * result. -+ * -+ * The @c _mali_uk_gp_job_finished_s contains the job status (see \ref _mali_uk_job_status), -+ * the number of milliseconds the job took to render, and values of core registers -+ * when the job finished (irq status, performance counters, renderer list -+ * address). A job has finished succesfully when its status is -+ * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering -+ * the job, or software detected the job is taking more than watchdog_msecs to -+ * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. -+ * If the hardware detected a bus error while accessing memory associated with the -+ * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. -+ * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to -+ * stop the job but the job didn't start on the hardware yet, e.g. when the -+ * driver shutdown. -+ * -+ * In case the job got suspended, @c _mali_uk_gp_job_suspended_s contains -+ * the @c user_job_ptr identifier used to start the job with, the @c reason -+ * why the job stalled (see \ref _maligp_job_suspended_reason) and a @c cookie -+ * to identify the core on which the job stalled. This @c cookie will be needed -+ * when responding to this nofication by means of _mali_ukk_gp_suspend_response(). -+ * (see _mali_ukk_gp_suspend_response()). The response is either to abort or -+ * resume the job. If the job got suspended due to an out of memory condition -+ * you may be able to resolve this by providing more memory and resuming the job. -+ * -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 user_job_ptr; /**< [in] identifier for the job in user space, a @c mali_gp_job_info* */ -+ u32 priority; /**< [in] job priority. A lower number means higher priority */ -+ u32 frame_registers[MALIGP2_NUM_REGS_FRAME]; /**< [in] core specific registers associated with this job */ -+ u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ -+ u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ -+ u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ -+ u32 frame_builder_id; /**< [in] id of the originating frame builder */ -+ u32 flush_id; /**< [in] flush id within the originating frame builder */ -+ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ -+ u64 timeline_point_ptr; /**< [in,out] pointer to u32: location where point on gp timeline for this job will be written */ -+ u32 varying_memsize; /** < [in] size of varying memory to use deffer bind*/ -+ u32 deferred_mem_num; -+ u64 deferred_mem_list; /** < [in] memory hanlde list of varying buffer to use deffer bind */ -+} _mali_uk_gp_start_job_s; -+ -+#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE (1<<0) /**< Enable performance counter SRC0 for a job */ -+#define _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE (1<<1) /**< Enable performance counter SRC1 for a job */ -+#define _MALI_PERFORMANCE_COUNTER_FLAG_HEATMAP_ENABLE (1<<2) /**< Enable per tile (aka heatmap) generation with for a job (using the enabled counter sources) */ -+ -+/** @} */ /* end group _mali_uk_gpstartjob_s */ -+ -+typedef struct { -+ u64 user_job_ptr; /**< [out] identifier for the job in user space */ -+ _mali_uk_job_status status; /**< [out] status of finished job */ -+ u32 heap_current_addr; /**< [out] value of the GP PLB PL heap start address register */ -+ u32 perf_counter0; /**< [out] value of performance counter 0 (see ARM DDI0415A) */ -+ u32 perf_counter1; /**< [out] value of performance counter 1 (see ARM DDI0415A) */ -+ u32 pending_big_job_num; -+} _mali_uk_gp_job_finished_s; -+ -+typedef struct { -+ u64 user_job_ptr; /**< [out] identifier for the job in user space */ -+ u32 cookie; /**< [out] identifier for the core in kernel space on which the job stalled */ -+} _mali_uk_gp_job_suspended_s; -+ -+/** @} */ /* end group _mali_uk_gp */ -+ -+ -+/** @defgroup _mali_uk_pp U/K Fragment Processor -+ * @{ */ -+ -+#define _MALI_PP_MAX_SUB_JOBS 8 -+ -+#define _MALI_PP_MAX_FRAME_REGISTERS ((0x058/4)+1) -+ -+#define _MALI_PP_MAX_WB_REGISTERS ((0x02C/4)+1) -+ -+#define _MALI_DLBU_MAX_REGISTERS 4 -+ -+/** Flag for _mali_uk_pp_start_job_s */ -+#define _MALI_PP_JOB_FLAG_NO_NOTIFICATION (1<<0) -+#define _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE (1<<1) -+#define _MALI_PP_JOB_FLAG_PROTECTED (1<<2) -+ -+/** @defgroup _mali_uk_ppstartjob_s Fragment Processor Start Job -+ * @{ */ -+ -+/** @brief Arguments for _mali_ukk_pp_start_job() -+ * -+ * To start a Fragment Processor job -+ * - associate the request with a reference to a mali_pp_job by setting -+ * @c user_job_ptr to the address of the @c mali_pp_job of the job. -+ * - set @c priority to the priority of the mali_pp_job -+ * - specify a timeout for the job by setting @c watchdog_msecs to the number of -+ * milliseconds the job is allowed to run. Specifying a value of 0 selects the -+ * default timeout in use by the device driver. -+ * - copy the frame registers from the @c mali_pp_job into @c frame_registers. -+ * For MALI200 you also need to copy the write back 0,1 and 2 registers. -+ * - set the @c perf_counter_flag, @c perf_counter_src0 and @c perf_counter_src1 to zero -+ * for a non-instrumented build. For an instrumented build you can use up -+ * to two performance counters. Set the corresponding bit in @c perf_counter_flag -+ * to enable them. @c perf_counter_src0 and @c perf_counter_src1 specify -+ * the source of what needs to get counted (e.g. number of vertex loader -+ * cache hits). For source id values, see ARM DDI0415A, Table 3-60. -+ * - pass in the user-kernel context in @c ctx that was returned from _mali_ukk_open() -+ * -+ * When _mali_ukk_pp_start_job() returns @c _MALI_OSK_ERR_OK, @c status contains the -+ * result of the request (see \ref _mali_uk_start_job_status). If the job could -+ * not get started (@c _MALI_UK_START_JOB_NOT_STARTED_DO_REQUEUE) it should be -+ * tried again. -+ * -+ * After the job has started, _mali_wait_for_notification() will be notified -+ * when the job finished. The notification will contain a -+ * @c _mali_uk_pp_job_finished_s result. It contains the @c user_job_ptr -+ * identifier used to start the job with, the job @c status (see \ref _mali_uk_job_status), -+ * the number of milliseconds the job took to render, and values of core registers -+ * when the job finished (irq status, performance counters, renderer list -+ * address). A job has finished succesfully when its status is -+ * @c _MALI_UK_JOB_STATUS_FINISHED. If the hardware detected a timeout while rendering -+ * the job, or software detected the job is taking more than @c watchdog_msecs to -+ * complete, the status will indicate @c _MALI_UK_JOB_STATUS_HANG. -+ * If the hardware detected a bus error while accessing memory associated with the -+ * job, status will indicate @c _MALI_UK_JOB_STATUS_SEG_FAULT. -+ * status will indicate @c _MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to -+ * stop the job but the job didn't start on the hardware yet, e.g. when the -+ * driver shutdown. -+ * -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 user_job_ptr; /**< [in] identifier for the job in user space */ -+ u32 priority; /**< [in] job priority. A lower number means higher priority */ -+ u32 frame_registers[_MALI_PP_MAX_FRAME_REGISTERS]; /**< [in] core specific registers associated with first sub job, see ARM DDI0415A */ -+ u32 frame_registers_addr_frame[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_FRAME registers for sub job 1-7 */ -+ u32 frame_registers_addr_stack[_MALI_PP_MAX_SUB_JOBS - 1]; /**< [in] ADDR_STACK registers for sub job 1-7 */ -+ u32 wb0_registers[_MALI_PP_MAX_WB_REGISTERS]; -+ u32 wb1_registers[_MALI_PP_MAX_WB_REGISTERS]; -+ u32 wb2_registers[_MALI_PP_MAX_WB_REGISTERS]; -+ u32 dlbu_registers[_MALI_DLBU_MAX_REGISTERS]; /**< [in] Dynamic load balancing unit registers */ -+ u32 num_cores; /**< [in] Number of cores to set up (valid range: 1-8(M450) or 4(M400)) */ -+ u32 perf_counter_flag; /**< [in] bitmask indicating which performance counters to enable, see \ref _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE and related macro definitions */ -+ u32 perf_counter_src0; /**< [in] source id for performance counter 0 (see ARM DDI0415A, Table 3-60) */ -+ u32 perf_counter_src1; /**< [in] source id for performance counter 1 (see ARM DDI0415A, Table 3-60) */ -+ u32 frame_builder_id; /**< [in] id of the originating frame builder */ -+ u32 flush_id; /**< [in] flush id within the originating frame builder */ -+ u32 flags; /**< [in] See _MALI_PP_JOB_FLAG_* for a list of avaiable flags */ -+ u32 tilesx; /**< [in] number of tiles in the x direction (needed for heatmap generation */ -+ u32 tilesy; /**< [in] number of tiles in y direction (needed for reading the heatmap memory) */ -+ u32 heatmap_mem; /**< [in] memory address to store counter values per tile (aka heatmap) */ -+ u32 num_memory_cookies; /**< [in] number of memory cookies attached to job */ -+ u64 memory_cookies; /**< [in] pointer to array of u32 memory cookies attached to job */ -+ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ -+ u64 timeline_point_ptr; /**< [in,out] pointer to location of u32 where point on pp timeline for this job will be written */ -+} _mali_uk_pp_start_job_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 gp_args; /**< [in,out] GP uk arguments (see _mali_uk_gp_start_job_s) */ -+ u64 pp_args; /**< [in,out] PP uk arguments (see _mali_uk_pp_start_job_s) */ -+} _mali_uk_pp_and_gp_start_job_s; -+ -+/** @} */ /* end group _mali_uk_ppstartjob_s */ -+ -+typedef struct { -+ u64 user_job_ptr; /**< [out] identifier for the job in user space */ -+ _mali_uk_job_status status; /**< [out] status of finished job */ -+ u32 perf_counter0[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 0 (see ARM DDI0415A), one for each sub job */ -+ u32 perf_counter1[_MALI_PP_MAX_SUB_JOBS]; /**< [out] value of perfomance counter 1 (see ARM DDI0415A), one for each sub job */ -+ u32 perf_counter_src0; -+ u32 perf_counter_src1; -+} _mali_uk_pp_job_finished_s; -+ -+typedef struct { -+ u32 number_of_enabled_cores; /**< [out] the new number of enabled cores */ -+} _mali_uk_pp_num_cores_changed_s; -+ -+ -+ -+/** -+ * Flags to indicate write-back units -+ */ -+typedef enum { -+ _MALI_UK_PP_JOB_WB0 = 1, -+ _MALI_UK_PP_JOB_WB1 = 2, -+ _MALI_UK_PP_JOB_WB2 = 4, -+} _mali_uk_pp_job_wbx_flag; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 fb_id; /**< [in] Frame builder ID of job to disable WB units for */ -+ u32 wb0_memory; -+ u32 wb1_memory; -+ u32 wb2_memory; -+} _mali_uk_pp_disable_wb_s; -+ -+ -+/** @} */ /* end group _mali_uk_pp */ -+ -+/** @defgroup _mali_uk_soft_job U/K Soft Job -+ * @{ */ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 user_job; /**< [in] identifier for the job in user space */ -+ u64 job_id_ptr; /**< [in,out] pointer to location of u32 where job id will be written */ -+ _mali_uk_fence_t fence; /**< [in] fence this job must wait on */ -+ u32 point; /**< [out] point on soft timeline for this job */ -+ u32 type; /**< [in] type of soft job */ -+} _mali_uk_soft_job_start_s; -+ -+typedef struct { -+ u64 user_job; /**< [out] identifier for the job in user space */ -+} _mali_uk_soft_job_activated_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 job_id; /**< [in] id for soft job */ -+} _mali_uk_soft_job_signal_s; -+ -+/** @} */ /* end group _mali_uk_soft_job */ -+ -+typedef struct { -+ u32 counter_id; -+ u32 key; -+ int enable; -+} _mali_uk_annotate_profiling_mem_counter_s; -+ -+typedef struct { -+ u32 sampling_rate; -+ int enable; -+} _mali_uk_annotate_profiling_enable_s; -+ -+ -+/** @addtogroup _mali_uk_core U/K Core -+ * @{ */ -+ -+/** @defgroup _mali_uk_waitfornotification_s Wait For Notification -+ * @{ */ -+ -+/** @brief Notification type encodings -+ * -+ * Each Notification type is an ordered pair of (subsystem,id), and is unique. -+ * -+ * The encoding of subsystem,id into a 32-bit word is: -+ * encoding = (( subsystem << _MALI_NOTIFICATION_SUBSYSTEM_SHIFT ) & _MALI_NOTIFICATION_SUBSYSTEM_MASK) -+ * | (( id << _MALI_NOTIFICATION_ID_SHIFT ) & _MALI_NOTIFICATION_ID_MASK) -+ * -+ * @see _mali_uk_wait_for_notification_s -+ */ -+typedef enum { -+ /** core notifications */ -+ -+ _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x20, -+ _MALI_NOTIFICATION_APPLICATION_QUIT = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x40, -+ _MALI_NOTIFICATION_SETTINGS_CHANGED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x80, -+ _MALI_NOTIFICATION_SOFT_ACTIVATED = (_MALI_UK_CORE_SUBSYSTEM << 16) | 0x100, -+ -+ /** Fragment Processor notifications */ -+ -+ _MALI_NOTIFICATION_PP_FINISHED = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x10, -+ _MALI_NOTIFICATION_PP_NUM_CORE_CHANGE = (_MALI_UK_PP_SUBSYSTEM << 16) | 0x20, -+ -+ /** Vertex Processor notifications */ -+ -+ _MALI_NOTIFICATION_GP_FINISHED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x10, -+ _MALI_NOTIFICATION_GP_STALLED = (_MALI_UK_GP_SUBSYSTEM << 16) | 0x20, -+ -+ /** Profiling notifications */ -+ _MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x10, -+ _MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE = (_MALI_UK_PROFILING_SUBSYSTEM << 16) | 0x20, -+} _mali_uk_notification_type; -+ -+/** to assist in splitting up 32-bit notification value in subsystem and id value */ -+#define _MALI_NOTIFICATION_SUBSYSTEM_MASK 0xFFFF0000 -+#define _MALI_NOTIFICATION_SUBSYSTEM_SHIFT 16 -+#define _MALI_NOTIFICATION_ID_MASK 0x0000FFFF -+#define _MALI_NOTIFICATION_ID_SHIFT 0 -+ -+ -+/** @brief Enumeration of possible settings which match mali_setting_t in user space -+ * -+ * -+ */ -+typedef enum { -+ _MALI_UK_USER_SETTING_SW_EVENTS_ENABLE = 0, -+ _MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, -+ _MALI_UK_USER_SETTING_DEPTHBUFFER_CAPTURE_ENABLED, -+ _MALI_UK_USER_SETTING_STENCILBUFFER_CAPTURE_ENABLED, -+ _MALI_UK_USER_SETTING_PER_TILE_COUNTERS_CAPTURE_ENABLED, -+ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_COMPOSITOR, -+ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_WINDOW, -+ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_OTHER, -+ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, -+ _MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, -+ _MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, -+ _MALI_UK_USER_SETTING_MAX, -+} _mali_uk_user_setting_t; -+ -+/* See mali_user_settings_db.c */ -+extern const char *_mali_uk_user_setting_descriptions[]; -+#define _MALI_UK_USER_SETTING_DESCRIPTIONS \ -+ { \ -+ "sw_events_enable", \ -+ "colorbuffer_capture_enable", \ -+ "depthbuffer_capture_enable", \ -+ "stencilbuffer_capture_enable", \ -+ "per_tile_counters_enable", \ -+ "buffer_capture_compositor", \ -+ "buffer_capture_window", \ -+ "buffer_capture_other", \ -+ "buffer_capture_n_frames", \ -+ "buffer_capture_resize_factor", \ -+ "sw_counters_enable", \ -+ }; -+ -+/** @brief struct to hold the value to a particular setting as seen in the kernel space -+ */ -+typedef struct { -+ _mali_uk_user_setting_t setting; -+ u32 value; -+} _mali_uk_settings_changed_s; -+ -+/** @brief Arguments for _mali_ukk_wait_for_notification() -+ * -+ * On successful return from _mali_ukk_wait_for_notification(), the members of -+ * this structure will indicate the reason for notification. -+ * -+ * Specifically, the source of the notification can be identified by the -+ * subsystem and id fields of the mali_uk_notification_type in the code.type -+ * member. The type member is encoded in a way to divide up the types into a -+ * subsystem field, and a per-subsystem ID field. See -+ * _mali_uk_notification_type for more information. -+ * -+ * Interpreting the data union member depends on the notification type: -+ * -+ * - type == _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS -+ * - The kernel side is shutting down. No further -+ * _mali_uk_wait_for_notification() calls should be made. -+ * - In this case, the value of the data union member is undefined. -+ * - This is used to indicate to the user space client that it should close -+ * the connection to the Mali Device Driver. -+ * - type == _MALI_NOTIFICATION_PP_FINISHED -+ * - The notification data is of type _mali_uk_pp_job_finished_s. It contains the user_job_ptr -+ * identifier used to start the job with, the job status, the number of milliseconds the job took to render, -+ * and values of core registers when the job finished (irq status, performance counters, renderer list -+ * address). -+ * - A job has finished succesfully when its status member is _MALI_UK_JOB_STATUS_FINISHED. -+ * - If the hardware detected a timeout while rendering the job, or software detected the job is -+ * taking more than watchdog_msecs (see _mali_ukk_pp_start_job()) to complete, the status member will -+ * indicate _MALI_UK_JOB_STATUS_HANG. -+ * - If the hardware detected a bus error while accessing memory associated with the job, status will -+ * indicate _MALI_UK_JOB_STATUS_SEG_FAULT. -+ * - Status will indicate MALI_UK_JOB_STATUS_NOT_STARTED if the driver had to stop the job but the job -+ * didn't start the hardware yet, e.g. when the driver closes. -+ * - type == _MALI_NOTIFICATION_GP_FINISHED -+ * - The notification data is of type _mali_uk_gp_job_finished_s. The notification is similar to that of -+ * type == _MALI_NOTIFICATION_PP_FINISHED, except that several other GP core register values are returned. -+ * The status values have the same meaning for type == _MALI_NOTIFICATION_PP_FINISHED. -+ * - type == _MALI_NOTIFICATION_GP_STALLED -+ * - The nofication data is of type _mali_uk_gp_job_suspended_s. It contains the user_job_ptr -+ * identifier used to start the job with, the reason why the job stalled and a cookie to identify the core on -+ * which the job stalled. -+ * - The reason member of gp_job_suspended is set to _MALIGP_JOB_SUSPENDED_OUT_OF_MEMORY -+ * when the polygon list builder unit has run out of memory. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_notification_type type; /**< [out] Type of notification available */ -+ union { -+ _mali_uk_gp_job_suspended_s gp_job_suspended;/**< [out] Notification data for _MALI_NOTIFICATION_GP_STALLED notification type */ -+ _mali_uk_gp_job_finished_s gp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_GP_FINISHED notification type */ -+ _mali_uk_pp_job_finished_s pp_job_finished; /**< [out] Notification data for _MALI_NOTIFICATION_PP_FINISHED notification type */ -+ _mali_uk_settings_changed_s setting_changed;/**< [out] Notification data for _MALI_NOTIFICAATION_SETTINGS_CHANGED notification type */ -+ _mali_uk_soft_job_activated_s soft_job_activated; /**< [out] Notification data for _MALI_NOTIFICATION_SOFT_ACTIVATED notification type */ -+ _mali_uk_annotate_profiling_mem_counter_s profiling_mem_counter; -+ _mali_uk_annotate_profiling_enable_s profiling_enable; -+ } data; -+} _mali_uk_wait_for_notification_s; -+ -+/** @brief Arguments for _mali_ukk_post_notification() -+ * -+ * Posts the specified notification to the notification queue for this application. -+ * This is used to send a quit message to the callback thread. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_notification_type type; /**< [in] Type of notification to post */ -+} _mali_uk_post_notification_s; -+ -+/** @} */ /* end group _mali_uk_waitfornotification_s */ -+ -+/** @defgroup _mali_uk_getapiversion_s Get API Version -+ * @{ */ -+ -+/** helpers for Device Driver API version handling */ -+ -+/** @brief Encode a version ID from a 16-bit input -+ * -+ * @note the input is assumed to be 16 bits. It must not exceed 16 bits. */ -+#define _MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) -+ -+/** @brief Check whether a 32-bit value is likely to be Device Driver API -+ * version ID. */ -+#define _IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) -+ -+/** @brief Decode a 16-bit version number from a 32-bit Device Driver API version -+ * ID */ -+#define _GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) -+ -+/** @brief Determine whether two 32-bit encoded version IDs match */ -+#define _IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) -+ /** -+ * RK MALI version code -+ */ -+#define _MALI_RK_LIBS_VERSION 1 -+ -+/** -+ * API version define. -+ * Indicates the version of the kernel API -+ * The version is a 16bit integer incremented on each API change. -+ * The 16bit integer is stored twice in a 32bit integer -+ * For example, for version 1 the value would be 0x00010001 -+ */ -+#define _MALI_API_VERSION 900 -+#define _MALI_UK_API_VERSION _MAKE_VERSION_ID(_MALI_API_VERSION) -+ -+/** -+ * The API version is a 16-bit integer stored in both the lower and upper 16-bits -+ * of a 32-bit value. The 16-bit API version value is incremented on each API -+ * change. Version 1 would be 0x00010001. Used in _mali_uk_get_api_version_s. -+ */ -+typedef u32 _mali_uk_api_version; -+ -+/** @brief Arguments for _mali_uk_get_api_version() -+ * -+ * The user-side interface version must be written into the version member, -+ * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of -+ * the kernel-side interface. -+ * -+ * On successful return, the version member will be the API version of the -+ * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version -+ * of the API. -+ * -+ * The compatible member must be checked to see if the version of the user-side -+ * interface is compatible with the kernel-side interface, since future versions -+ * of the interface may be backwards compatible. -+ */ -+typedef struct { -+ u32 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ -+ int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ -+} _mali_uk_get_api_version_s; -+ -+/** @brief Arguments for _mali_uk_get_api_version_v2() -+ * -+ * The user-side interface version must be written into the version member, -+ * encoded using _MAKE_VERSION_ID(). It will be compared to the API version of -+ * the kernel-side interface. -+ * -+ * On successful return, the version member will be the API version of the -+ * kernel-side interface. _MALI_UK_API_VERSION macro defines the current version -+ * of the API. -+ * -+ * The compatible member must be checked to see if the version of the user-side -+ * interface is compatible with the kernel-side interface, since future versions -+ * of the interface may be backwards compatible. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ -+ int compatible; /**< [out] @c 1 when @version is compatible, @c 0 otherwise */ -+} _mali_uk_get_api_version_v2_s; -+ -+typedef struct -+{ -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ -+} _mali_uk_get_mali_version_in_rk30_s; -+ -+/* rk_ext : rk_ko_ver_t. */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_api_version version; /**< [in,out] API version of user-side interface. */ -+} _mali_rk_ko_version_s; -+/** @} */ /* end group _mali_uk_getapiversion_s */ -+ -+/** @defgroup _mali_uk_get_user_settings_s Get user space settings */ -+ -+/** @brief struct to keep the matching values of the user space settings within certain context -+ * -+ * Each member of the settings array corresponds to a matching setting in the user space and its value is the value -+ * of that particular setting. -+ * -+ * All settings are given reference to the context pointed to by the ctx pointer. -+ * -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 settings[_MALI_UK_USER_SETTING_MAX]; /**< [out] The values for all settings */ -+} _mali_uk_get_user_settings_s; -+ -+/** @brief struct to hold the value of a particular setting from the user space within a given context -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_user_setting_t setting; /**< [in] setting to get */ -+ u32 value; /**< [out] value of setting */ -+} _mali_uk_get_user_setting_s; -+ -+/** @brief Arguments for _mali_ukk_request_high_priority() */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+} _mali_uk_request_high_priority_s; -+ -+/** @brief Arguments for _mali_ukk_pending_submit() */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+} _mali_uk_pending_submit_s; -+ -+/** @} */ /* end group _mali_uk_core */ -+ -+ -+/** @defgroup _mali_uk_memory U/K Memory -+ * @{ */ -+ -+#define _MALI_MEMORY_ALLOCATE_RESIZEABLE (1<<4) /* BUFFER can trim dow/grow*/ -+#define _MALI_MEMORY_ALLOCATE_NO_BIND_GPU (1<<5) /*Not map to GPU when allocate, must call bind later*/ -+#define _MALI_MEMORY_ALLOCATE_SWAPPABLE (1<<6) /* Allocate swappale memory. */ -+#define _MALI_MEMORY_ALLOCATE_DEFER_BIND (1<<7) /*Not map to GPU when allocate, must call bind later*/ -+#define _MALI_MEMORY_ALLOCATE_SECURE (1<<8) /* Allocate secure memory. */ -+ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 gpu_vaddr; /**< [in] GPU virtual address */ -+ u32 vsize; /**< [in] vitrual size of the allocation */ -+ u32 psize; /**< [in] physical size of the allocation */ -+ u32 flags; -+ u64 backend_handle; /**< [out] backend handle */ -+ s32 secure_shared_fd; /** < [in] the mem handle for secure mem */ -+} _mali_uk_alloc_mem_s; -+ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 gpu_vaddr; /**< [in] use as handle to free allocation */ -+ u32 free_pages_nr; /** < [out] record the number of free pages */ -+} _mali_uk_free_mem_s; -+ -+ -+#define _MALI_MEMORY_BIND_BACKEND_UMP (1<<8) -+#define _MALI_MEMORY_BIND_BACKEND_DMA_BUF (1<<9) -+#define _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY (1<<10) -+#define _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY (1<<11) -+#define _MALI_MEMORY_BIND_BACKEND_EXT_COW (1<<12) -+#define _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION (1<<13) -+ -+ -+#define _MALI_MEMORY_BIND_BACKEND_MASK (_MALI_MEMORY_BIND_BACKEND_UMP| \ -+ _MALI_MEMORY_BIND_BACKEND_DMA_BUF |\ -+ _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY |\ -+ _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY |\ -+ _MALI_MEMORY_BIND_BACKEND_EXT_COW |\ -+ _MALI_MEMORY_BIND_BACKEND_HAVE_ALLOCATION) -+ -+ -+#define _MALI_MEMORY_GPU_READ_ALLOCATE (1<<16) -+ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 vaddr; /**< [in] mali address to map the physical memory to */ -+ u32 size; /**< [in] size */ -+ u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ -+ u32 padding; /** padding for 32/64 struct alignment */ -+ union { -+ struct { -+ u32 secure_id; /**< [in] secure id */ -+ u32 rights; /**< [in] rights necessary for accessing memory */ -+ u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ -+ } bind_ump; -+ struct { -+ u32 mem_fd; /**< [in] Memory descriptor */ -+ u32 rights; /**< [in] rights necessary for accessing memory */ -+ u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ -+ } bind_dma_buf; -+ struct { -+ u32 phys_addr; /**< [in] physical address */ -+ u32 rights; /**< [in] rights necessary for accessing memory */ -+ u32 flags; /**< [in] flags, see \ref _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE */ -+ } bind_ext_memory; -+ } mem_union; -+} _mali_uk_bind_mem_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 flags; /**< [in] see_MALI_MEMORY_BIND_BACKEND_* */ -+ u32 vaddr; /**< [in] identifier for mapped memory object in kernel space */ -+} _mali_uk_unbind_mem_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 target_handle; /**< [in] handle of allocation need to do COW */ -+ u32 target_offset; /**< [in] offset in target allocation to do COW(for support COW a memory allocated from memory_bank, PAGE_SIZE align)*/ -+ u32 target_size; /**< [in] size of target allocation to do COW (for support memory bank, PAGE_SIZE align)(in byte) */ -+ u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation (PAGE_SIZE align)*/ -+ u32 range_size; /**< [in] re allocate size (PAGE_SIZE align)*/ -+ u32 vaddr; /**< [in] mali address for the new allocaiton */ -+ u32 backend_handle; /**< [out] backend handle */ -+ u32 flags; -+} _mali_uk_cow_mem_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 range_start; /**< [in] re allocate range start offset, offset from the start of allocation */ -+ u32 size; /**< [in] re allocate size*/ -+ u32 vaddr; /**< [in] mali address for the new allocaiton */ -+ s32 change_pages_nr; /**< [out] record the page number change for cow operation */ -+} _mali_uk_cow_modify_range_s; -+ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 mem_fd; /**< [in] Memory descriptor */ -+ u32 size; /**< [out] size */ -+} _mali_uk_dma_buf_get_size_s; -+ -+/** Flag for _mali_uk_map_external_mem_s, _mali_uk_attach_ump_mem_s and _mali_uk_attach_dma_buf_s */ -+#define _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE (1<<0) -+ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 vaddr; /* the buffer to do resize*/ -+ u32 psize; /* wanted physical size of this memory */ -+} _mali_uk_mem_resize_s; -+ -+/** -+ * @brief Arguments for _mali_uk[uk]_mem_write_safe() -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 src; /**< [in] Pointer to source data */ -+ u64 dest; /**< [in] Destination Mali buffer */ -+ u32 size; /**< [in,out] Number of bytes to write/copy on input, number of bytes actually written/copied on output */ -+} _mali_uk_mem_write_safe_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 size; /**< [out] size of MMU page table information (registers + page tables) */ -+} _mali_uk_query_mmu_page_table_dump_size_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 size; /**< [in] size of buffer to receive mmu page table information */ -+ u64 buffer; /**< [in,out] buffer to receive mmu page table information */ -+ u32 register_writes_size; /**< [out] size of MMU register dump */ -+ u64 register_writes; /**< [out] pointer within buffer where MMU register dump is stored */ -+ u32 page_table_dump_size; /**< [out] size of MMU page table dump */ -+ u64 page_table_dump; /**< [out] pointer within buffer where MMU page table dump is stored */ -+} _mali_uk_dump_mmu_page_table_s; -+ -+/** @} */ /* end group _mali_uk_memory */ -+ -+ -+/** @addtogroup _mali_uk_pp U/K Fragment Processor -+ * @{ */ -+ -+/** @brief Arguments for _mali_ukk_get_pp_number_of_cores() -+ * -+ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() -+ * - Upon successful return from _mali_ukk_get_pp_number_of_cores(), @c number_of_cores -+ * will contain the number of Fragment Processor cores in the system. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 number_of_total_cores; /**< [out] Total number of Fragment Processor cores in the system */ -+ u32 number_of_enabled_cores; /**< [out] Number of enabled Fragment Processor cores */ -+} _mali_uk_get_pp_number_of_cores_s; -+ -+/** @brief Arguments for _mali_ukk_get_pp_core_version() -+ * -+ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() -+ * - Upon successful return from _mali_ukk_get_pp_core_version(), @c version contains -+ * the version that all Fragment Processor cores are compatible with. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ -+ u32 padding; -+} _mali_uk_get_pp_core_version_s; -+ -+/** @} */ /* end group _mali_uk_pp */ -+ -+ -+/** @addtogroup _mali_uk_gp U/K Vertex Processor -+ * @{ */ -+ -+/** @brief Arguments for _mali_ukk_get_gp_number_of_cores() -+ * -+ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() -+ * - Upon successful return from _mali_ukk_get_gp_number_of_cores(), @c number_of_cores -+ * will contain the number of Vertex Processor cores in the system. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 number_of_cores; /**< [out] number of Vertex Processor cores in the system */ -+} _mali_uk_get_gp_number_of_cores_s; -+ -+/** @brief Arguments for _mali_ukk_get_gp_core_version() -+ * -+ * - pass in the user-kernel context @c ctx that was returned from _mali_ukk_open() -+ * - Upon successful return from _mali_ukk_get_gp_core_version(), @c version contains -+ * the version that all Vertex Processor cores are compatible with. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_core_version version; /**< [out] version returned from core, see \ref _mali_core_version */ -+} _mali_uk_get_gp_core_version_s; -+ -+/** @} */ /* end group _mali_uk_gp */ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 event_id; /**< [in] event id to register (see enum mali_profiling_events for values) */ -+ u32 data[5]; /**< [in] event specific data */ -+} _mali_uk_profiling_add_event_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 memory_usage; /**< [out] total memory usage */ -+ u32 vaddr; /**< [in] mali address for the cow allocaiton */ -+ s32 change_pages_nr; /**< [out] record the page number change for cow operation */ -+} _mali_uk_profiling_memory_usage_get_s; -+ -+ -+/** @addtogroup _mali_uk_memory U/K Memory -+ * @{ */ -+ -+/** @brief Arguments to _mali_ukk_mem_mmap() -+ * -+ * Use of the phys_addr member depends on whether the driver is compiled for -+ * Mali-MMU or nonMMU: -+ * - in the nonMMU case, this is the physical address of the memory as seen by -+ * the CPU (which may be a constant offset from that used by Mali) -+ * - in the MMU case, this is the Mali Virtual base address of the memory to -+ * allocate, and the particular physical pages used to back the memory are -+ * entirely determined by _mali_ukk_mem_mmap(). The details of the physical pages -+ * are not reported to user-space for security reasons. -+ * -+ * The cookie member must be stored for use later when freeing the memory by -+ * calling _mali_ukk_mem_munmap(). In the Mali-MMU case, the cookie is secure. -+ * -+ * The ukk_private word must be set to zero when calling from user-space. On -+ * Kernel-side, the OS implementation of the U/K interface can use it to -+ * communicate data to the OS implementation of the OSK layer. In particular, -+ * _mali_ukk_get_big_block() directly calls _mali_ukk_mem_mmap directly, and -+ * will communicate its own ukk_private word through the ukk_private member -+ * here. The common code itself will not inspect or modify the ukk_private -+ * word, and so it may be safely used for whatever purposes necessary to -+ * integrate Mali Memory handling into the OS. -+ * -+ * The uku_private member is currently reserved for use by the user-side -+ * implementation of the U/K interface. Its value must be zero. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ void *mapping; /**< [out] Returns user-space virtual address for the mapping */ -+ u32 size; /**< [in] Size of the requested mapping */ -+ u32 phys_addr; /**< [in] Physical address - could be offset, depending on caller+callee convention */ -+ mali_bool writeable; -+} _mali_uk_mem_mmap_s; -+ -+/** @brief Arguments to _mali_ukk_mem_munmap() -+ * -+ * The cookie and mapping members must be that returned from the same previous -+ * call to _mali_ukk_mem_mmap(). The size member must correspond to cookie -+ * and mapping - that is, it must be the value originally supplied to a call to -+ * _mali_ukk_mem_mmap that returned the values of mapping and cookie. -+ * -+ * An error will be returned if an attempt is made to unmap only part of the -+ * originally obtained range, or to unmap more than was originally obtained. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ void *mapping; /**< [in] The mapping returned from mmap call */ -+ u32 size; /**< [in] The size passed to mmap call */ -+} _mali_uk_mem_munmap_s; -+/** @} */ /* end group _mali_uk_memory */ -+ -+/** @defgroup _mali_uk_vsync U/K VSYNC Wait Reporting Module -+ * @{ */ -+ -+/** @brief VSYNC events -+ * -+ * These events are reported when DDK starts to wait for vsync and when the -+ * vsync has occured and the DDK can continue on the next frame. -+ */ -+typedef enum _mali_uk_vsync_event { -+ _MALI_UK_VSYNC_EVENT_BEGIN_WAIT = 0, -+ _MALI_UK_VSYNC_EVENT_END_WAIT -+} _mali_uk_vsync_event; -+ -+/** @brief Arguments to _mali_ukk_vsync_event() -+ * -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_vsync_event event; /**< [in] VSYNCH event type */ -+} _mali_uk_vsync_event_report_s; -+ -+/** @} */ /* end group _mali_uk_vsync */ -+ -+/** @defgroup _mali_uk_sw_counters_report U/K Software Counter Reporting -+ * @{ */ -+ -+/** @brief Software counter values -+ * -+ * Values recorded for each of the software counters during a single renderpass. -+ */ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 counters; /**< [in] The array of u32 counter values */ -+ u32 num_counters; /**< [in] The number of elements in counters array */ -+} _mali_uk_sw_counters_report_s; -+ -+/** @} */ /* end group _mali_uk_sw_counters_report */ -+ -+/** @defgroup _mali_uk_timeline U/K Mali Timeline -+ * @{ */ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 timeline; /**< [in] timeline id */ -+ u32 point; /**< [out] latest point on timeline */ -+} _mali_uk_timeline_get_latest_point_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_fence_t fence; /**< [in] fence */ -+ u32 timeout; /**< [in] timeout (0 for no wait, -1 for blocking) */ -+ u32 status; /**< [out] status of fence (1 if signaled, 0 if timeout) */ -+} _mali_uk_timeline_wait_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ _mali_uk_fence_t fence; /**< [in] mali fence to create linux sync fence from */ -+ s32 sync_fd; /**< [out] file descriptor for new linux sync fence */ -+} _mali_uk_timeline_create_sync_fence_s; -+ -+/** @} */ /* end group _mali_uk_timeline */ -+ -+/** @} */ /* end group u_k_api */ -+ -+/** @} */ /* end group uddapi */ -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ s32 stream_fd; /**< [in] The profiling kernel base stream fd handle */ -+} _mali_uk_profiling_stream_fd_get_s; -+ -+typedef struct { -+ u64 ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u64 control_packet_data; /**< [in] the control packet data for control settings */ -+ u32 control_packet_size; /**< [in] The control packet size */ -+ u64 response_packet_data; /** < [out] The response packet data */ -+ u32 response_packet_size; /** < [in,out] The response packet data */ -+} _mali_uk_profiling_control_set_s; -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_UTGARD_UK_TYPES_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h -new file mode 100755 -index 000000000..6fafc6777 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/license/gpl/mali_kernel_license.h -@@ -0,0 +1,30 @@ -+/* -+ * Copyright (C) 2010, 2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_kernel_license.h -+ * Defines for the macro MODULE_LICENSE. -+ */ -+ -+#ifndef __MALI_KERNEL_LICENSE_H__ -+#define __MALI_KERNEL_LICENSE_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#define MALI_KERNEL_LINUX_LICENSE "GPL" -+#define MALI_LICENSE_IS_GPL 1 -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_KERNEL_LICENSE_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c -new file mode 100755 -index 000000000..260c2a822 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.c -@@ -0,0 +1,354 @@ -+/* -+ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_osk_mali.h" -+#include "mali_kernel_common.h" -+ -+#include -+#include -+#include -+#include -+#ifdef CONFIG_DEVFREQ_THERMAL -+#include -+#endif -+ -+#include -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) -+#include -+#else /* Linux >= 3.13 */ -+/* In 3.13 the OPP include header file, types, and functions were all -+ * renamed. Use the old filename for the include, and define the new names to -+ * the old, when an old kernel is detected. -+ */ -+#include -+#define dev_pm_opp opp -+#define dev_pm_opp_get_voltage opp_get_voltage -+#define dev_pm_opp_get_opp_count opp_get_opp_count -+#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil -+#endif /* Linux >= 3.13 */ -+ -+#include "mali_pm_metrics.h" -+ -+#include -+#include -+ -+static struct monitor_dev_profile mali_mdevp = { -+ .type = MONITOR_TPYE_DEV, -+ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, -+ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, -+}; -+ -+static struct devfreq_simple_ondemand_data ondemand_data; -+ -+static int -+mali_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) -+{ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ struct dev_pm_opp *opp; -+ unsigned long freq = 0; -+ unsigned long old_freq = mdev->current_freq; -+ unsigned long voltage; -+ int err; -+ -+ freq = *target_freq; -+ -+ opp = devfreq_recommended_opp(dev, &freq, flags); -+ if (IS_ERR(opp)) { -+ MALI_PRINT_ERROR(("Failed to get opp (%ld)\n", PTR_ERR(opp))); -+ return PTR_ERR(opp); -+ } -+ voltage = dev_pm_opp_get_voltage(opp); -+ dev_pm_opp_put(opp); -+ -+ MALI_DEBUG_PRINT(2, ("mali_devfreq_target:set_freq = %lld flags = 0x%x\n", freq, flags)); -+ /* -+ * Only update if there is a change of frequency -+ */ -+ if (old_freq == freq) { -+ *target_freq = freq; -+ mali_pm_reset_dvfs_utilisation(mdev); -+#ifdef CONFIG_REGULATOR -+ if (mdev->current_voltage == voltage) -+ return 0; -+ err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); -+ if (err) { -+ dev_err(dev, "Failed to set voltage (%d)\n", err); -+ return err; -+ } -+ mdev->current_voltage = voltage; -+#endif -+ return 0; -+ } -+ -+#ifdef CONFIG_REGULATOR -+ if (mdev->regulator && mdev->current_voltage != voltage && -+ old_freq < freq) { -+ err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); -+ if (err) { -+ MALI_PRINT_ERROR(("Failed to increase voltage (%d)\n", err)); -+ return err; -+ } -+ } -+#endif -+ -+ err = clk_set_rate(mdev->clock, freq); -+ if (err) { -+ MALI_PRINT_ERROR(("Failed to set clock %lu (target %lu)\n", freq, *target_freq)); -+ return err; -+ } -+ *target_freq = freq; -+ mdev->current_freq = freq; -+ if (mdev->devfreq) -+ mdev->devfreq->last_status.current_frequency = freq; -+ -+#ifdef CONFIG_REGULATOR -+ if (mdev->regulator && mdev->current_voltage != voltage && -+ old_freq > freq) { -+ err = regulator_set_voltage(mdev->regulator, voltage, INT_MAX); -+ if (err) { -+ MALI_PRINT_ERROR(("Failed to decrease voltage (%d)\n", err)); -+ return err; -+ } -+ } -+#endif -+ -+ mdev->current_voltage = voltage; -+ -+ mali_pm_reset_dvfs_utilisation(mdev); -+ -+ return err; -+} -+ -+static int -+mali_devfreq_cur_freq(struct device *dev, unsigned long *freq) -+{ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ *freq = mdev->current_freq; -+ -+ MALI_DEBUG_PRINT(2, ("mali_devfreq_cur_freq: freq = %d \n", *freq)); -+ return 0; -+} -+ -+static int -+mali_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) -+{ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ stat->current_frequency = mdev->current_freq; -+ -+ mali_pm_get_dvfs_utilisation(mdev, -+ &stat->total_time, &stat->busy_time); -+ -+ stat->private_data = NULL; -+ -+#ifdef CONFIG_DEVFREQ_THERMAL -+ memcpy(&mdev->devfreq->last_status, stat, sizeof(*stat)); -+#endif -+ -+ return 0; -+} -+ -+/* setup platform specific opp in platform.c*/ -+int __weak setup_opps(void) -+{ -+ return 0; -+} -+ -+/* term platform specific opp in platform.c*/ -+int __weak term_opps(struct device *dev) -+{ -+ return 0; -+} -+ -+static int mali_devfreq_init_freq_table(struct mali_device *mdev, -+ struct devfreq_dev_profile *dp) -+{ -+ int err, count; -+ int i = 0; -+ unsigned long freq = 0; -+ struct dev_pm_opp *opp; -+ -+ err = setup_opps(); -+ if (err) -+ return err; -+ -+ count = dev_pm_opp_get_opp_count(mdev->dev); -+ if (count < 0) { -+ return count; -+ } -+ -+ MALI_DEBUG_PRINT(2, ("mali devfreq table count %d\n", count)); -+ -+ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), -+ GFP_KERNEL); -+ if (!dp->freq_table) -+ return -ENOMEM; -+ -+ for (i = 0; i < count; i++, freq++) { -+ opp = dev_pm_opp_find_freq_ceil(mdev->dev, &freq); -+ if (IS_ERR(opp)) -+ break; -+ dev_pm_opp_put(opp); -+ -+ dp->freq_table[i] = freq; -+ MALI_DEBUG_PRINT(2, ("mali devfreq table array[%d] = %d\n", i, freq)); -+ } -+ -+ if (count != i) -+ MALI_PRINT_ERROR(("Unable to enumerate all OPPs (%d!=%d)\n", -+ count, i)); -+ -+ dp->max_state = i; -+ -+ return 0; -+} -+ -+static void mali_devfreq_term_freq_table(struct mali_device *mdev) -+{ -+ struct devfreq_dev_profile *dp = mdev->devfreq->profile; -+ -+ kfree(dp->freq_table); -+ term_opps(mdev->dev); -+} -+ -+static void mali_devfreq_exit(struct device *dev) -+{ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ mali_devfreq_term_freq_table(mdev); -+} -+ -+int mali_devfreq_init(struct mali_device *mdev) -+{ -+ struct device_node *np = mdev->dev->of_node; -+#ifdef CONFIG_DEVFREQ_THERMAL -+ struct devfreq_cooling_power *callbacks = NULL; -+ _mali_osk_device_data data; -+#endif -+ struct devfreq_dev_profile *dp; -+ struct dev_pm_opp *opp; -+ unsigned long opp_rate; -+ int err; -+ -+ MALI_DEBUG_PRINT(2, ("Init Mali devfreq\n")); -+ -+ if (!mdev->clock) -+ return -ENODEV; -+ -+ mdev->current_freq = clk_get_rate(mdev->clock); -+ -+ dp = &mdev->devfreq_profile; -+ -+ dp->initial_freq = mdev->current_freq; -+ dp->polling_ms = 100; -+ dp->target = mali_devfreq_target; -+ dp->get_dev_status = mali_devfreq_status; -+ dp->get_cur_freq = mali_devfreq_cur_freq; -+ dp->exit = mali_devfreq_exit; -+ -+ if (mali_devfreq_init_freq_table(mdev, dp)) -+ return -EFAULT; -+ -+ of_property_read_u32(np, "upthreshold", -+ &ondemand_data.upthreshold); -+ of_property_read_u32(np, "downdifferential", -+ &ondemand_data.downdifferential); -+ -+ mdev->devfreq = devfreq_add_device(mdev->dev, dp, -+ "simple_ondemand", &ondemand_data); -+ if (IS_ERR(mdev->devfreq)) { -+ mali_devfreq_term_freq_table(mdev); -+ return PTR_ERR(mdev->devfreq); -+ } -+ -+ err = devfreq_register_opp_notifier(mdev->dev, mdev->devfreq); -+ if (err) { -+ MALI_PRINT_ERROR(("Failed to register OPP notifier (%d)\n", err)); -+ goto opp_notifier_failed; -+ } -+ -+ opp_rate = mdev->current_freq; -+ opp = devfreq_recommended_opp(mdev->dev, &opp_rate, 0); -+ if (!IS_ERR(opp)) -+ dev_pm_opp_put(opp); -+ mdev->devfreq->last_status.current_frequency = opp_rate; -+ -+ mali_mdevp.data = mdev->devfreq; -+ mdev->mdev_info = rockchip_system_monitor_register(mdev->dev, -+ &mali_mdevp); -+ if (IS_ERR(mdev->mdev_info)) { -+ dev_dbg(mdev->dev, "without system monitor\n"); -+ mdev->mdev_info = NULL; -+ } -+#ifdef CONFIG_DEVFREQ_THERMAL -+ if (of_machine_is_compatible("rockchip,rk3036")) -+ return 0; -+ -+ /* Initilization last_status it will be used when first power allocate called */ -+ mdev->devfreq->last_status.current_frequency = mdev->current_freq; -+ -+ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { -+ if (NULL != data.gpu_cooling_ops) { -+ callbacks = data.gpu_cooling_ops; -+ MALI_DEBUG_PRINT(2, ("Mali GPU Thermal: Callback handler installed \n")); -+ } -+ } -+ -+ if (callbacks) { -+ mdev->devfreq_cooling = of_devfreq_cooling_register_power( -+ mdev->dev->of_node, -+ mdev->devfreq, -+ callbacks); -+ if (IS_ERR_OR_NULL(mdev->devfreq_cooling)) { -+ err = PTR_ERR(mdev->devfreq_cooling); -+ MALI_PRINT_ERROR(("Failed to register cooling device (%d)\n", err)); -+ goto cooling_failed; -+ } else { -+ MALI_DEBUG_PRINT(2, ("Mali GPU Thermal Cooling installed \n")); -+ } -+ } -+#endif -+ -+ return 0; -+ -+#ifdef CONFIG_DEVFREQ_THERMAL -+cooling_failed: -+ devfreq_unregister_opp_notifier(mdev->dev, mdev->devfreq); -+#endif /* CONFIG_DEVFREQ_THERMAL */ -+opp_notifier_failed: -+ err = devfreq_remove_device(mdev->devfreq); -+ if (err) -+ MALI_PRINT_ERROR(("Failed to terminate devfreq (%d)\n", err)); -+ else -+ mdev->devfreq = NULL; -+ -+ return err; -+} -+ -+void mali_devfreq_term(struct mali_device *mdev) -+{ -+ int err; -+ -+ MALI_DEBUG_PRINT(2, ("Term Mali devfreq\n")); -+ -+ rockchip_system_monitor_unregister(mdev->mdev_info); -+#ifdef CONFIG_DEVFREQ_THERMAL -+ devfreq_cooling_unregister(mdev->devfreq_cooling); -+#endif -+ -+ devfreq_unregister_opp_notifier(mdev->dev, mdev->devfreq); -+ -+ err = devfreq_remove_device(mdev->devfreq); -+ if (err) -+ MALI_PRINT_ERROR(("Failed to terminate devfreq (%d)\n", err)); -+ else -+ mdev->devfreq = NULL; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h -new file mode 100755 -index 000000000..ba7c017d8 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_devfreq.h -@@ -0,0 +1,17 @@ -+/* -+ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#ifndef _MALI_DEVFREQ_H_ -+#define _MALI_DEVFREQ_H_ -+ -+int mali_devfreq_init(struct mali_device *mdev); -+ -+void mali_devfreq_term(struct mali_device *mdev); -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c -new file mode 100755 -index 000000000..95c3ea12d ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_device_pause_resume.c -@@ -0,0 +1,36 @@ -+/** -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_device_pause_resume.c -+ * Implementation of the Mali pause/resume functionality -+ */ -+ -+#include -+#include -+#include "mali_pm.h" -+ -+void mali_dev_pause(void) -+{ -+ /* -+ * Deactive all groups to prevent hardware being touched -+ * during the period of mali device pausing -+ */ -+ mali_pm_os_suspend(MALI_FALSE); -+} -+ -+EXPORT_SYMBOL(mali_dev_pause); -+ -+void mali_dev_resume(void) -+{ -+ mali_pm_os_resume(); -+} -+ -+EXPORT_SYMBOL(mali_dev_resume); -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c -new file mode 100755 -index 000000000..e026e11e4 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.c -@@ -0,0 +1,439 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) -+#include "mali_dma_fence.h" -+#include -+#include -+#endif -+ -+static DEFINE_SPINLOCK(mali_dma_fence_lock); -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static bool mali_dma_fence_enable_signaling(struct dma_fence *fence) -+{ -+ MALI_IGNORE(fence); -+ return true; -+} -+ -+static const char *mali_dma_fence_get_driver_name(struct dma_fence *fence) -+{ -+ MALI_IGNORE(fence); -+ return "mali"; -+} -+ -+static const char *mali_dma_fence_get_timeline_name(struct dma_fence *fence) -+{ -+ MALI_IGNORE(fence); -+ return "mali_dma_fence"; -+} -+ -+static const struct dma_fence_ops mali_dma_fence_ops = { -+ .get_driver_name = mali_dma_fence_get_driver_name, -+ .get_timeline_name = mali_dma_fence_get_timeline_name, -+ .enable_signaling = mali_dma_fence_enable_signaling, -+ .signaled = NULL, -+ .wait = dma_fence_default_wait, -+ .release = NULL -+}; -+#else -+static bool mali_dma_fence_enable_signaling(struct fence *fence) -+{ -+ MALI_IGNORE(fence); -+ return true; -+} -+ -+static const char *mali_dma_fence_get_driver_name(struct fence *fence) -+{ -+ MALI_IGNORE(fence); -+ return "mali"; -+} -+ -+static const char *mali_dma_fence_get_timeline_name(struct fence *fence) -+{ -+ MALI_IGNORE(fence); -+ return "mali_dma_fence"; -+} -+ -+static const struct fence_ops mali_dma_fence_ops = { -+ .get_driver_name = mali_dma_fence_get_driver_name, -+ .get_timeline_name = mali_dma_fence_get_timeline_name, -+ .enable_signaling = mali_dma_fence_enable_signaling, -+ .signaled = NULL, -+ .wait = fence_default_wait, -+ .release = NULL -+}; -+#endif -+ -+static void mali_dma_fence_context_cleanup(struct mali_dma_fence_context *dma_fence_context) -+{ -+ u32 i; -+ -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ -+ for (i = 0; i < dma_fence_context->num_dma_fence_waiter; i++) { -+ if (dma_fence_context->mali_dma_fence_waiters[i]) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence, -+ &dma_fence_context->mali_dma_fence_waiters[i]->base); -+ dma_fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence); -+ -+#else -+ fence_remove_callback(dma_fence_context->mali_dma_fence_waiters[i]->fence, -+ &dma_fence_context->mali_dma_fence_waiters[i]->base); -+ fence_put(dma_fence_context->mali_dma_fence_waiters[i]->fence); -+#endif -+ kfree(dma_fence_context->mali_dma_fence_waiters[i]); -+ dma_fence_context->mali_dma_fence_waiters[i] = NULL; -+ } -+ } -+ -+ if (NULL != dma_fence_context->mali_dma_fence_waiters) -+ kfree(dma_fence_context->mali_dma_fence_waiters); -+ -+ dma_fence_context->mali_dma_fence_waiters = NULL; -+ dma_fence_context->num_dma_fence_waiter = 0; -+} -+ -+static void mali_dma_fence_context_work_func(struct work_struct *work_handle) -+{ -+ struct mali_dma_fence_context *dma_fence_context; -+ -+ MALI_DEBUG_ASSERT_POINTER(work_handle); -+ -+ dma_fence_context = container_of(work_handle, struct mali_dma_fence_context, work_handle); -+ -+ dma_fence_context->cb_func(dma_fence_context->pp_job_ptr); -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static void mali_dma_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb) -+#else -+static void mali_dma_fence_callback(struct fence *fence, struct fence_cb *cb) -+#endif -+{ -+ struct mali_dma_fence_waiter *dma_fence_waiter = NULL; -+ struct mali_dma_fence_context *dma_fence_context = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ MALI_DEBUG_ASSERT_POINTER(cb); -+ -+ MALI_IGNORE(fence); -+ -+ dma_fence_waiter = container_of(cb, struct mali_dma_fence_waiter, base); -+ dma_fence_context = dma_fence_waiter->parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ -+ if (atomic_dec_and_test(&dma_fence_context->count)) -+ schedule_work(&dma_fence_context->work_handle); -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct dma_fence *fence) -+#else -+static _mali_osk_errcode_t mali_dma_fence_add_callback(struct mali_dma_fence_context *dma_fence_context, struct fence *fence) -+#endif -+{ -+ int ret = 0; -+ struct mali_dma_fence_waiter *dma_fence_waiter; -+ struct mali_dma_fence_waiter **dma_fence_waiters; -+ -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ dma_fence_waiters = krealloc(dma_fence_context->mali_dma_fence_waiters, -+ (dma_fence_context->num_dma_fence_waiter + 1) -+ * sizeof(struct mali_dma_fence_waiter *), -+ GFP_KERNEL); -+ -+ if (NULL == dma_fence_waiters) { -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to realloc the dma fence waiters.\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ dma_fence_context->mali_dma_fence_waiters = dma_fence_waiters; -+ -+ dma_fence_waiter = kzalloc(sizeof(struct mali_dma_fence_waiter), GFP_KERNEL); -+ -+ if (NULL == dma_fence_waiter) { -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create mali dma fence waiter.\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_get(fence); -+#else -+ fence_get(fence); -+#endif -+ dma_fence_waiter->fence = fence; -+ dma_fence_waiter->parent = dma_fence_context; -+ atomic_inc(&dma_fence_context->count); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ ret = dma_fence_add_callback(fence, &dma_fence_waiter->base, -+ mali_dma_fence_callback); -+#else -+ ret = fence_add_callback(fence, &dma_fence_waiter->base, -+ mali_dma_fence_callback); -+#endif -+ if (0 > ret) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_put(fence); -+#else -+ fence_put(fence); -+#endif -+ kfree(dma_fence_waiter); -+ atomic_dec(&dma_fence_context->count); -+ if (-ENOENT == ret) { -+ /*-ENOENT if fence has already been signaled, return _MALI_OSK_ERR_OK*/ -+ return _MALI_OSK_ERR_OK; -+ } -+ /* Failed to add the fence callback into fence, return _MALI_OSK_ERR_FAULT*/ -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into fence.\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ dma_fence_context->mali_dma_fence_waiters[dma_fence_context->num_dma_fence_waiter] = dma_fence_waiter; -+ dma_fence_context->num_dma_fence_waiter++; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+struct dma_fence *mali_dma_fence_new(u32 context, u32 seqno) -+#else -+struct fence *mali_dma_fence_new(u32 context, u32 seqno) -+#endif -+{ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ struct dma_fence *fence = NULL; -+ fence = kzalloc(sizeof(struct dma_fence), GFP_KERNEL); -+#else -+ struct fence *fence = NULL; -+ fence = kzalloc(sizeof(struct fence), GFP_KERNEL); -+#endif -+ if (NULL == fence) { -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to create dma fence.\n")); -+ return fence; -+ } -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_init(fence, -+ &mali_dma_fence_ops, -+ &mali_dma_fence_lock, -+ context, seqno); -+#else -+ fence_init(fence, -+ &mali_dma_fence_ops, -+ &mali_dma_fence_lock, -+ context, seqno); -+#endif -+ return fence; -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+void mali_dma_fence_signal_and_put(struct dma_fence **fence) -+#else -+void mali_dma_fence_signal_and_put(struct fence **fence) -+#endif -+{ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ MALI_DEBUG_ASSERT_POINTER(*fence); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_signal(*fence); -+ dma_fence_put(*fence); -+#else -+ fence_signal(*fence); -+ fence_put(*fence); -+#endif -+ *fence = NULL; -+} -+ -+void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context, -+ mali_dma_fence_context_callback_func_t cb_func, -+ void *pp_job_ptr) -+{ -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ -+ INIT_WORK(&dma_fence_context->work_handle, mali_dma_fence_context_work_func); -+ atomic_set(&dma_fence_context->count, 1); -+ dma_fence_context->num_dma_fence_waiter = 0; -+ dma_fence_context->mali_dma_fence_waiters = NULL; -+ dma_fence_context->cb_func = cb_func; -+ dma_fence_context->pp_job_ptr = pp_job_ptr; -+} -+ -+_mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context, -+ struct reservation_object *dma_reservation_object) -+{ -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; -+ u32 shared_count = 0, i; -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ struct dma_fence *exclusive_fence = NULL; -+ struct dma_fence **shared_fences = NULL; -+#else -+ struct fence *exclusive_fence = NULL; -+ struct fence **shared_fences = NULL; -+#endif -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object); -+ -+ /* Get all the shared/exclusive fences in the reservation object of dma buf*/ -+ ret = reservation_object_get_fences_rcu(dma_reservation_object, &exclusive_fence, -+ &shared_count, &shared_fences); -+ if (ret < 0) { -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to get shared or exclusive_fence dma fences from the reservation object of dma buf.\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ if (exclusive_fence) { -+ ret = mali_dma_fence_add_callback(dma_fence_context, exclusive_fence); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into exclusive fence.\n")); -+ mali_dma_fence_context_cleanup(dma_fence_context); -+ goto ended; -+ } -+ } -+ -+ -+ for (i = 0; i < shared_count; i++) { -+ ret = mali_dma_fence_add_callback(dma_fence_context, shared_fences[i]); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to add callback into shared fence [%d].\n", i)); -+ mali_dma_fence_context_cleanup(dma_fence_context); -+ break; -+ } -+ } -+ -+ended: -+ -+ if (exclusive_fence) -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_put(exclusive_fence); -+#else -+ fence_put(exclusive_fence); -+#endif -+ -+ if (shared_fences) { -+ for (i = 0; i < shared_count; i++) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_put(shared_fences[i]); -+#else -+ fence_put(shared_fences[i]); -+#endif -+ } -+ kfree(shared_fences); -+ } -+ -+ return ret; -+} -+ -+ -+void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context) -+{ -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ atomic_set(&dma_fence_context->count, 0); -+ if (dma_fence_context->work_handle.func) { -+ cancel_work_sync(&dma_fence_context->work_handle); -+ } -+ mali_dma_fence_context_cleanup(dma_fence_context); -+} -+ -+void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context) -+{ -+ MALI_DEBUG_ASSERT_POINTER(dma_fence_context); -+ -+ if (atomic_dec_and_test(&dma_fence_context->count)) -+ schedule_work(&dma_fence_context->work_handle); -+} -+ -+ -+void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object, -+ struct reservation_object **dma_reservation_object_list, -+ u32 *num_dma_reservation_object) -+{ -+ u32 i; -+ -+ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object); -+ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list); -+ MALI_DEBUG_ASSERT_POINTER(num_dma_reservation_object); -+ -+ for (i = 0; i < *num_dma_reservation_object; i++) { -+ if (dma_reservation_object_list[i] == dma_reservation_object) -+ return; -+ } -+ -+ dma_reservation_object_list[*num_dma_reservation_object] = dma_reservation_object; -+ (*num_dma_reservation_object)++; -+} -+ -+int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list, -+ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx) -+{ -+ u32 i; -+ -+ struct reservation_object *reservation_object_to_slow_lock = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(dma_reservation_object_list); -+ MALI_DEBUG_ASSERT_POINTER(ww_actx); -+ -+ ww_acquire_init(ww_actx, &reservation_ww_class); -+ -+again: -+ for (i = 0; i < num_dma_reservation_object; i++) { -+ int ret; -+ -+ if (dma_reservation_object_list[i] == reservation_object_to_slow_lock) { -+ reservation_object_to_slow_lock = NULL; -+ continue; -+ } -+ -+ ret = ww_mutex_lock(&dma_reservation_object_list[i]->lock, ww_actx); -+ -+ if (ret < 0) { -+ u32 slow_lock_index = i; -+ -+ /* unlock all pre locks we have already locked.*/ -+ while (i > 0) { -+ i--; -+ ww_mutex_unlock(&dma_reservation_object_list[i]->lock); -+ } -+ -+ if (NULL != reservation_object_to_slow_lock) -+ ww_mutex_unlock(&reservation_object_to_slow_lock->lock); -+ -+ if (ret == -EDEADLK) { -+ reservation_object_to_slow_lock = dma_reservation_object_list[slow_lock_index]; -+ ww_mutex_lock_slow(&reservation_object_to_slow_lock->lock, ww_actx); -+ goto again; -+ } -+ ww_acquire_fini(ww_actx); -+ MALI_DEBUG_PRINT(1, ("Mali dma fence: failed to lock all dma reservation objects.\n", i)); -+ return ret; -+ } -+ } -+ -+ ww_acquire_done(ww_actx); -+ return 0; -+} -+ -+void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list, -+ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx) -+{ -+ u32 i; -+ -+ for (i = 0; i < num_dma_reservation_object; i++) -+ ww_mutex_unlock(&dma_reservation_object_list[i]->lock); -+ -+ ww_acquire_fini(ww_actx); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h -new file mode 100755 -index 000000000..d44f6d1a8 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_dma_fence.h -@@ -0,0 +1,124 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_dma_fence.h -+ * -+ * Mali interface for Linux dma buf fence objects. -+ */ -+ -+#ifndef _MALI_DMA_FENCE_H_ -+#define _MALI_DMA_FENCE_H_ -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#include -+#else -+#include -+#endif -+#include -+#endif -+ -+struct mali_dma_fence_context; -+ -+/* The mali dma fence context callback function */ -+typedef void (*mali_dma_fence_context_callback_func_t)(void *pp_job_ptr); -+ -+struct mali_dma_fence_waiter { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ struct dma_fence *fence; -+ struct dma_fence_cb base; -+#else -+ struct fence_cb base; -+ struct fence *fence; -+#endif -+ struct mali_dma_fence_context *parent; -+}; -+ -+struct mali_dma_fence_context { -+ struct work_struct work_handle; -+ struct mali_dma_fence_waiter **mali_dma_fence_waiters; -+ u32 num_dma_fence_waiter; -+ atomic_t count; -+ void *pp_job_ptr; /* the mali pp job pointer */; -+ mali_dma_fence_context_callback_func_t cb_func; -+}; -+ -+/* Create a dma fence -+ * @param context The execution context this fence is run on -+ * @param seqno A linearly increasing sequence number for this context -+ * @return the new dma fence if success, or NULL on failure. -+ */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+struct dma_fence *mali_dma_fence_new(u32 context, u32 seqno); -+#else -+struct fence *mali_dma_fence_new(u32 context, u32 seqno); -+#endif -+/* Signal and put dma fence -+ * @param fence The dma fence to signal and put -+ */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+void mali_dma_fence_signal_and_put(struct dma_fence **fence); -+#else -+void mali_dma_fence_signal_and_put(struct fence **fence); -+#endif -+/** -+ * Initialize a mali dma fence context for pp job. -+ * @param dma_fence_context The mali dma fence context to initialize. -+ * @param cb_func The dma fence context callback function to call when all dma fence release. -+ * @param pp_job_ptr The pp_job to call function with. -+ */ -+void mali_dma_fence_context_init(struct mali_dma_fence_context *dma_fence_context, -+ mali_dma_fence_context_callback_func_t cb_func, -+ void *pp_job_ptr); -+ -+/** -+ * Add new mali dma fence waiter into mali dma fence context -+ * @param dma_fence_context The mali dma fence context -+ * @param dma_reservation_object the reservation object to create new mali dma fence waiters -+ * @return _MALI_OSK_ERR_OK if success, or not. -+ */ -+_mali_osk_errcode_t mali_dma_fence_context_add_waiters(struct mali_dma_fence_context *dma_fence_context, -+ struct reservation_object *dma_reservation_object); -+ -+/** -+ * Release the dma fence context -+ * @param dma_fence_text The mali dma fence context. -+ */ -+void mali_dma_fence_context_term(struct mali_dma_fence_context *dma_fence_context); -+ -+/** -+ * Decrease the dma fence context atomic count -+ * @param dma_fence_text The mali dma fence context. -+ */ -+void mali_dma_fence_context_dec_count(struct mali_dma_fence_context *dma_fence_context); -+ -+/** -+ * Get all reservation object -+ * @param dma_reservation_object The reservation object to add into the reservation object list -+ * @param dma_reservation_object_list The reservation object list to store all reservation object -+ * @param num_dma_reservation_object The number of all reservation object -+ */ -+void mali_dma_fence_add_reservation_object_list(struct reservation_object *dma_reservation_object, -+ struct reservation_object **dma_reservation_object_list, -+ u32 *num_dma_reservation_object); -+ -+/** -+ * Wait/wound mutex lock to lock all reservation object. -+ */ -+int mali_dma_fence_lock_reservation_object_list(struct reservation_object **dma_reservation_object_list, -+ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx); -+ -+/** -+ * Wait/wound mutex lock to unlock all reservation object. -+ */ -+void mali_dma_fence_unlock_reservation_object_list(struct reservation_object **dma_reservation_object_list, -+ u32 num_dma_reservation_object, struct ww_acquire_ctx *ww_actx); -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c -new file mode 100755 -index 000000000..e13cbad3e ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.c -@@ -0,0 +1,783 @@ -+/* -+ * Copyright (C) 2012-2018 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_internal_sync.h" -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#if defined(DEBUG) -+#include "mali_session.h" -+#include "mali_timeline.h" -+#endif -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static const struct dma_fence_ops fence_ops; -+#else -+static const struct fence_ops fence_ops; -+#endif -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static struct mali_internal_sync_point *mali_internal_fence_to_sync_pt(struct dma_fence *fence) -+#else -+static struct mali_internal_sync_point *mali_internal_fence_to_sync_pt(struct fence *fence) -+#endif -+{ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ return container_of(fence, struct mali_internal_sync_point, base); -+} -+ -+static inline struct mali_internal_sync_timeline *mali_internal_sync_pt_to_sync_timeline(struct mali_internal_sync_point *sync_pt) -+{ -+ MALI_DEBUG_ASSERT_POINTER(sync_pt); -+ return container_of(sync_pt->base.lock, struct mali_internal_sync_timeline, sync_pt_list_lock); -+} -+ -+static void mali_internal_sync_timeline_free(struct kref *kref_count) -+{ -+ struct mali_internal_sync_timeline *sync_timeline; -+ -+ MALI_DEBUG_ASSERT_POINTER(kref_count); -+ -+ sync_timeline = container_of(kref_count, struct mali_internal_sync_timeline, kref_count); -+ -+ if (sync_timeline->ops->release_obj) -+ sync_timeline->ops->release_obj(sync_timeline); -+ -+ kfree(sync_timeline); -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+static void mali_internal_fence_check_cb_func(struct fence *fence, struct fence_cb *cb) -+#else -+static void mali_internal_fence_check_cb_func(struct dma_fence *fence, struct dma_fence_cb *cb) -+#endif -+{ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ struct mali_internal_sync_fence_cb *check; -+#else -+ struct mali_internal_sync_fence_waiter *waiter; -+#endif -+ struct mali_internal_sync_fence *sync_fence; -+ int ret; -+ MALI_DEBUG_ASSERT_POINTER(cb); -+ MALI_IGNORE(fence); -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ check = container_of(cb, struct mali_internal_sync_fence_cb, cb); -+ sync_fence = check->sync_file; -+#else -+ waiter = container_of(cb, struct mali_internal_sync_fence_waiter, cb); -+ sync_fence = (struct mali_internal_sync_fence *)waiter->work.private; -+#endif -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ ret = atomic_dec_and_test(&sync_fence->status); -+ if (ret) -+ wake_up_all(&sync_fence->wq); -+#else -+ ret = sync_fence->fence->ops->signaled(sync_fence->fence); -+ -+ if (0 > ret) -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to wait fence 0x%x for sync_fence 0x%x.\n", fence, sync_fence)); -+ if (1 == ret) -+ wake_up_all(&sync_fence->wq); -+#endif -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+static void mali_internal_sync_fence_add_fence(struct mali_internal_sync_fence *sync_fence, struct fence *sync_pt) -+{ -+ int fence_num = 0; -+ MALI_DEBUG_ASSERT_POINTER(sync_fence); -+ MALI_DEBUG_ASSERT_POINTER(sync_pt); -+ -+ fence_num = sync_fence->num_fences; -+ -+ sync_fence->cbs[fence_num].fence = sync_pt; -+ sync_fence->cbs[fence_num].sync_file = sync_fence; -+ -+ if (!fence_add_callback(sync_pt, &sync_fence->cbs[fence_num].cb, mali_internal_fence_check_cb_func)) { -+ fence_get(sync_pt); -+ sync_fence->num_fences++; -+ atomic_inc(&sync_fence->status); -+ } -+} -+#endif -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) -+static int mali_internal_sync_fence_wake_up_wq(wait_queue_entry_t *curr, unsigned mode, -+ int wake_flags, void *key) -+#else -+static int mali_internal_sync_fence_wake_up_wq(wait_queue_t *curr, unsigned mode, -+ int wake_flags, void *key) -+#endif -+{ -+ struct mali_internal_sync_fence_waiter *wait; -+ MALI_IGNORE(mode); -+ MALI_IGNORE(wake_flags); -+ MALI_IGNORE(key); -+ -+ wait = container_of(curr, struct mali_internal_sync_fence_waiter, work); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) -+ list_del_init(&wait->work.entry); -+#else -+ list_del_init(&wait->work.task_list); -+#endif -+ wait->callback(wait->work.private, wait); -+ return 1; -+} -+ -+struct mali_internal_sync_timeline *mali_internal_sync_timeline_create(const struct mali_internal_sync_timeline_ops *ops, -+ int size, const char *name) -+{ -+ struct mali_internal_sync_timeline *sync_timeline = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(ops); -+ -+ if (size < sizeof(struct mali_internal_sync_timeline)) { -+ MALI_PRINT_ERROR(("Mali internal sync:Invalid size to create the mali internal sync timeline.\n")); -+ goto err; -+ } -+ -+ sync_timeline = kzalloc(size, GFP_KERNEL); -+ if (NULL == sync_timeline) { -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to allocate buffer for the mali internal sync timeline.\n")); -+ goto err; -+ } -+ kref_init(&sync_timeline->kref_count); -+ sync_timeline->ops = ops; -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ sync_timeline->fence_context = dma_fence_context_alloc(1); -+#else -+ sync_timeline->fence_context = fence_context_alloc(1); -+#endif -+ strlcpy(sync_timeline->name, name, sizeof(sync_timeline->name)); -+ -+ INIT_LIST_HEAD(&sync_timeline->sync_pt_list_head); -+ spin_lock_init(&sync_timeline->sync_pt_list_lock); -+ -+ return sync_timeline; -+err: -+ if (NULL != sync_timeline) { -+ kfree(sync_timeline); -+ } -+ return NULL; -+} -+ -+void mali_internal_sync_timeline_destroy(struct mali_internal_sync_timeline *sync_timeline) -+{ -+ MALI_DEBUG_ASSERT_POINTER(sync_timeline); -+ -+ sync_timeline->destroyed = MALI_TRUE; -+ -+ smp_wmb(); -+ -+ mali_internal_sync_timeline_signal(sync_timeline); -+ kref_put(&sync_timeline->kref_count, mali_internal_sync_timeline_free); -+} -+ -+void mali_internal_sync_timeline_signal(struct mali_internal_sync_timeline *sync_timeline) -+{ -+ unsigned long flags; -+ struct mali_internal_sync_point *sync_pt, *next; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_timeline); -+ -+ spin_lock_irqsave(&sync_timeline->sync_pt_list_lock, flags); -+ -+ list_for_each_entry_safe(sync_pt, next, &sync_timeline->sync_pt_list_head, -+ sync_pt_list) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ if (dma_fence_is_signaled_locked(&sync_pt->base)) -+#else -+ if (fence_is_signaled_locked(&sync_pt->base)) -+#endif -+ list_del_init(&sync_pt->sync_pt_list); -+ } -+ -+ spin_unlock_irqrestore(&sync_timeline->sync_pt_list_lock, flags); -+} -+ -+struct mali_internal_sync_point *mali_internal_sync_point_create(struct mali_internal_sync_timeline *sync_timeline, int size) -+{ -+ unsigned long flags; -+ struct mali_internal_sync_point *sync_pt = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_timeline); -+ -+ if (size < sizeof(struct mali_internal_sync_point)) { -+ MALI_PRINT_ERROR(("Mali internal sync:Invalid size to create the mali internal sync point.\n")); -+ goto err; -+ } -+ -+ sync_pt = kzalloc(size, GFP_KERNEL); -+ if (NULL == sync_pt) { -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to allocate buffer for the mali internal sync point.\n")); -+ goto err; -+ } -+ spin_lock_irqsave(&sync_timeline->sync_pt_list_lock, flags); -+ kref_get(&sync_timeline->kref_count); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_init(&sync_pt->base, &fence_ops, &sync_timeline->sync_pt_list_lock, -+ sync_timeline->fence_context, ++sync_timeline->value); -+#else -+ fence_init(&sync_pt->base, &fence_ops, &sync_timeline->sync_pt_list_lock, -+ sync_timeline->fence_context, ++sync_timeline->value); -+#endif -+ INIT_LIST_HEAD(&sync_pt->sync_pt_list); -+ spin_unlock_irqrestore(&sync_timeline->sync_pt_list_lock, flags); -+ -+ return sync_pt; -+err: -+ if (NULL != sync_pt) { -+ kfree(sync_pt); -+ } -+ return NULL; -+} -+ -+struct mali_internal_sync_fence *mali_internal_sync_fence_fdget(int fd) -+{ -+ struct file *file = fget(fd); -+ -+ if (NULL == file) { -+ return NULL; -+ } -+ -+ return file->private_data; -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+struct mali_internal_sync_fence *mali_internal_sync_fence_merge( -+ struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) -+{ -+ struct mali_internal_sync_fence *new_sync_fence; -+ int i, j, num_fence1, num_fence2, total_fences; -+ struct fence *fence0 = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_fence1); -+ MALI_DEBUG_ASSERT_POINTER(sync_fence2); -+ -+ num_fence1 = sync_fence1->num_fences; -+ num_fence2 = sync_fence2->num_fences; -+ -+ total_fences = num_fence1 + num_fence2; -+ -+ i = 0; -+ j = 0; -+ -+ if (num_fence1 > 0) { -+ fence0 = sync_fence1->cbs[i].fence; -+ i = 1; -+ } else if (num_fence2 > 0) { -+ fence0 = sync_fence2->cbs[i].fence; -+ j = 1; -+ } -+ -+ new_sync_fence = (struct mali_internal_sync_fence *)sync_file_create(fence0); -+ if (NULL == new_sync_fence) { -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to create the mali internal sync fence when merging sync fence.\n")); -+ return NULL; -+ } -+ -+ fence_remove_callback(new_sync_fence->cb[0].fence, &new_sync_fence->cb[0].cb); -+ new_sync_fence->num_fences = 0; -+ atomic_dec(&new_sync_fence->status); -+ -+ for (; i < num_fence1 && j < num_fence2;) { -+ struct fence *fence1 = sync_fence1->cbs[i].fence; -+ struct fence *fence2 = sync_fence2->cbs[j].fence; -+ -+ if (fence1->context < fence2->context) { -+ mali_internal_sync_fence_add_fence(new_sync_fence, fence1); -+ -+ i++; -+ } else if (fence1->context > fence2->context) { -+ mali_internal_sync_fence_add_fence(new_sync_fence, fence2); -+ -+ j++; -+ } else { -+ if (fence1->seqno - fence2->seqno <= INT_MAX) -+ mali_internal_sync_fence_add_fence(new_sync_fence, fence1); -+ else -+ mali_internal_sync_fence_add_fence(new_sync_fence, fence2); -+ i++; -+ j++; -+ } -+ } -+ -+ for (; i < num_fence1; i++) -+ mali_internal_sync_fence_add_fence(new_sync_fence, sync_fence1->cbs[i].fence); -+ -+ for (; j < num_fence2; j++) -+ mali_internal_sync_fence_add_fence(new_sync_fence, sync_fence2->cbs[j].fence); -+ -+ return new_sync_fence; -+} -+#else -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+static struct fence **mali_internal_get_fences(struct mali_internal_sync_fence *sync_fence, int *num_fences) -+#else -+static struct dma_fence **mali_internal_get_fences(struct mali_internal_sync_fence *sync_fence, int *num_fences) -+#endif -+{ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ if (sync_fence->fence->ops == &fence_array_ops) { -+ struct fence_array *fence_array = container_of(sync_fence->fence, struct fence_array, base); -+ *num_fences = fence_array->num_fences; -+ return fence_array->fences; -+ } -+#else -+ if (sync_fence->fence->ops == &dma_fence_array_ops) { -+ struct dma_fence_array *fence_array = container_of(sync_fence->fence, struct dma_fence_array, base); -+ *num_fences = fence_array->num_fences; -+ return fence_array->fences; -+ } -+#endif -+ *num_fences = 1; -+ return &sync_fence->fence; -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+static void mali_internal_add_fence_array(struct fence **fences, int *num_fences, struct fence *fence) -+#else -+static void mali_internal_add_fence_array(struct dma_fence **fences, int *num_fences, struct dma_fence *fence) -+#endif -+{ -+ fences[*num_fences] = fence; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ if (!fence_is_signaled(fence)) { -+ fence_get(fence); -+ (*num_fences)++; -+ } -+#else -+ if (!dma_fence_is_signaled(fence)) { -+ dma_fence_get(fence); -+ (*num_fences)++; -+ } -+#endif -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+static int mali_internal_sync_fence_set_fence_array(struct mali_internal_sync_fence *sync_fence, -+ struct fence **fences, int num_fences) -+#else -+static int mali_internal_sync_fence_set_fence_array(struct mali_internal_sync_fence *sync_fence, -+ struct dma_fence **fences, int num_fences) -+#endif -+{ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ struct fence_array *array; -+#else -+ struct dma_fence_array *array; -+#endif -+ if(num_fences == 1) { -+ sync_fence->fence =fences[0]; -+ kfree(fences); -+ } else { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ array = fence_array_create(num_fences, fences, -+ fence_context_alloc(1), 1, false); -+#else -+ array = dma_fence_array_create(num_fences, fences, -+ dma_fence_context_alloc(1), 1, false); -+#endif -+ if (!array){ -+ return -ENOMEM; -+ } -+ sync_fence->fence = &array->base; -+ } -+ return 0; -+} -+ -+struct mali_internal_sync_fence *mali_internal_sync_fence_merge( -+ struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) -+{ -+ struct mali_internal_sync_fence *sync_fence; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ struct fence **fences, **nfences, **fences1, **fences2; -+#else -+ struct dma_fence **fences, **nfences, **fences1, **fences2; -+#endif -+ int real_num_fences, i, j, num_fences, num_fences1, num_fences2; -+ -+ fences1 = mali_internal_get_fences(sync_fence1, &num_fences1); -+ fences2 = mali_internal_get_fences(sync_fence2, &num_fences2); -+ -+ num_fences = num_fences1 + num_fences2; -+ -+ fences = kcalloc(num_fences, sizeof(*fences), GFP_KERNEL); -+ if (!fences) { -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to alloc buffer for fences.\n")); -+ goto fences_alloc_failed; -+ } -+ -+ for (real_num_fences = i = j = 0; i < num_fences1 && j < num_fences2;) { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ struct fence *fence1 = fences1[i]; -+ struct fence *fence2 = fences2[j]; -+#else -+ struct dma_fence *fence1 = fences1[i]; -+ struct dma_fence *fence2 = fences2[j]; -+#endif -+ if (fence1->context < fence2->context) { -+ mali_internal_add_fence_array(fences, &real_num_fences, fence1); -+ -+ i++; -+ } else if (fence1->context > fence2->context) { -+ mali_internal_add_fence_array(fences, &real_num_fences, fence2); -+ -+ j++; -+ } else { -+ if (fence1->seqno - fence2->seqno <= INT_MAX) -+ mali_internal_add_fence_array(fences, &real_num_fences, fence1); -+ else -+ mali_internal_add_fence_array(fences, &real_num_fences, fence2); -+ -+ i++; -+ j++; -+ } -+ } -+ -+ for (; i < num_fences1; i++) -+ mali_internal_add_fence_array(fences, &real_num_fences, fences1[i]); -+ -+ for (; j < num_fences2; j++) -+ mali_internal_add_fence_array(fences, &real_num_fences, fences2[j]); -+ -+ if (0 == real_num_fences) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ fences[real_num_fences++] = fence_get(fences1[0]); -+#else -+ fences[real_num_fences++] = dma_fence_get(fences1[0]); -+#endif -+ -+ if (num_fences > real_num_fences) { -+ nfences = krealloc(fences, real_num_fences * sizeof(*fences), -+ GFP_KERNEL); -+ if (!nfences) -+ goto nfences_alloc_failed; -+ -+ fences = nfences; -+ } -+ -+ sync_fence = (struct mali_internal_sync_fence *)sync_file_create(fences[0]); -+ if (NULL == sync_fence) { -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to create the mali internal sync fence when merging sync fence.\n")); -+ goto sync_fence_alloc_failed; -+ } -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ fence_put(fences[0]); -+#else -+ dma_fence_put(fences[0]); -+#endif -+ -+ if (mali_internal_sync_fence_set_fence_array(sync_fence, fences, real_num_fences) < 0) { -+ MALI_PRINT_ERROR(("Mali internal sync:Failed to set fence for sync fence.\n")); -+ goto sync_fence_set_failed; -+ } -+ -+ return sync_fence; -+ -+sync_fence_set_failed: -+ fput(sync_fence->file); -+sync_fence_alloc_failed: -+ for (i = 0; i < real_num_fences; i++) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ fence_put(fences[i]); -+#else -+ dma_fence_put(fences[i]); -+#endif -+nfences_alloc_failed: -+ kfree(fences); -+fences_alloc_failed: -+ return NULL; -+} -+#endif -+ -+void mali_internal_sync_fence_waiter_init(struct mali_internal_sync_fence_waiter *waiter, -+ mali_internal_sync_callback_t callback) -+{ -+ MALI_DEBUG_ASSERT_POINTER(waiter); -+ MALI_DEBUG_ASSERT_POINTER(callback); -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) -+ INIT_LIST_HEAD(&waiter->work.entry); -+#else -+ INIT_LIST_HEAD(&waiter->work.task_list); -+#endif -+ waiter->callback = callback; -+} -+ -+int mali_internal_sync_fence_wait_async(struct mali_internal_sync_fence *sync_fence, -+ struct mali_internal_sync_fence_waiter *waiter) -+{ -+ int err; -+ unsigned long flags; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_fence); -+ MALI_DEBUG_ASSERT_POINTER(waiter); -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ err = atomic_read(&sync_fence->status); -+ -+ if (0 > err) -+ return err; -+ -+ if (!err) -+ return 1; -+ -+ init_waitqueue_func_entry(&waiter->work, mali_internal_sync_fence_wake_up_wq); -+ waiter->work.private = sync_fence; -+ -+ spin_lock_irqsave(&sync_fence->wq.lock, flags); -+ err = atomic_read(&sync_fence->status); -+ -+ if (0 < err) -+ __add_wait_queue_tail(&sync_fence->wq, &waiter->work); -+ spin_unlock_irqrestore(&sync_fence->wq.lock, flags); -+ -+ if (0 > err) -+ return err; -+ -+ return !err; -+#else -+ if ((sync_fence->fence) && (sync_fence->fence->ops) && (sync_fence->fence->ops->signaled)) -+ err = sync_fence->fence->ops->signaled(sync_fence->fence); -+ else -+ err = -1; -+ -+ if (0 > err) -+ return err; -+ -+ if (1 == err) -+ return err; -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ err = dma_fence_add_callback(sync_fence->fence, &waiter->cb, mali_internal_fence_check_cb_func); -+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) -+ err = fence_add_callback(sync_fence->fence, &waiter->cb, mali_internal_fence_check_cb_func); -+#endif -+ -+ if (0 != err) { -+ if (-ENOENT == err) -+ err = 1; -+ return err; -+ } -+ init_waitqueue_func_entry(&waiter->work, mali_internal_sync_fence_wake_up_wq); -+ waiter->work.private = sync_fence; -+ -+ spin_lock_irqsave(&sync_fence->wq.lock, flags); -+ err = sync_fence->fence->ops->signaled(sync_fence->fence); -+ -+ if (0 == err){ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) -+ __add_wait_queue_entry_tail(&sync_fence->wq, &waiter->work); -+#else -+ __add_wait_queue_tail(&sync_fence->wq, &waiter->work); -+#endif -+ } -+ spin_unlock_irqrestore(&sync_fence->wq.lock, flags); -+ -+ return err; -+#endif -+} -+ -+int mali_internal_sync_fence_cancel_async(struct mali_internal_sync_fence *sync_fence, -+ struct mali_internal_sync_fence_waiter *waiter) -+{ -+ unsigned long flags; -+ int ret = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_fence); -+ MALI_DEBUG_ASSERT_POINTER(waiter); -+ -+ spin_lock_irqsave(&sync_fence->wq.lock, flags); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) -+ if (!list_empty(&waiter->work.entry)) -+ list_del_init(&waiter->work.entry); -+#else -+ if (!list_empty(&waiter->work.task_list)) -+ list_del_init(&waiter->work.task_list); -+#endif -+ else -+ ret = -ENOENT; -+ spin_unlock_irqrestore(&sync_fence->wq.lock, flags); -+ -+ if (0 == ret) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_remove_callback(sync_fence->fence, &waiter->cb); -+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) -+ fence_remove_callback(sync_fence->fence, &waiter->cb); -+#endif -+ -+ } -+ -+ return ret; -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static const char *mali_internal_fence_get_driver_name(struct dma_fence *fence) -+#else -+static const char *mali_internal_fence_get_driver_name(struct fence *fence) -+#endif -+{ -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_timeline *parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ sync_pt = mali_internal_fence_to_sync_pt(fence); -+ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); -+ -+ return parent->ops->driver_name; -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static const char *mali_internal_fence_get_timeline_name(struct dma_fence *fence) -+#else -+static const char *mali_internal_fence_get_timeline_name(struct fence *fence) -+#endif -+{ -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_timeline *parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ sync_pt = mali_internal_fence_to_sync_pt(fence); -+ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); -+ -+ return parent->name; -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static void mali_internal_fence_release(struct dma_fence *fence) -+#else -+static void mali_internal_fence_release(struct fence *fence) -+#endif -+{ -+ unsigned long flags; -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_timeline *parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ sync_pt = mali_internal_fence_to_sync_pt(fence); -+ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); -+ -+ -+ spin_lock_irqsave(fence->lock, flags); -+ if (WARN_ON_ONCE(!list_empty(&sync_pt->sync_pt_list))) -+ list_del(&sync_pt->sync_pt_list); -+ spin_unlock_irqrestore(fence->lock, flags); -+ -+ if (parent->ops->free_pt) -+ parent->ops->free_pt(sync_pt); -+ -+ kref_put(&parent->kref_count, mali_internal_sync_timeline_free); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_free(&sync_pt->base); -+#else -+ fence_free(&sync_pt->base); -+#endif -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static bool mali_internal_fence_signaled(struct dma_fence *fence) -+#else -+static bool mali_internal_fence_signaled(struct fence *fence) -+#endif -+{ -+ int ret; -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_timeline *parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ sync_pt = mali_internal_fence_to_sync_pt(fence); -+ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); -+ -+ ret = parent->ops->has_signaled(sync_pt); -+ if (0 > ret) -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) \ -+ || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 68))) -+ fence->error = ret; -+#else -+ fence->status = ret; -+#endif -+ return ret; -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static bool mali_internal_fence_enable_signaling(struct dma_fence *fence) -+#else -+static bool mali_internal_fence_enable_signaling(struct fence *fence) -+#endif -+{ -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_timeline *parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ -+ sync_pt = mali_internal_fence_to_sync_pt(fence); -+ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); -+ -+ if (mali_internal_fence_signaled(fence)) -+ return false; -+ -+ list_add_tail(&sync_pt->sync_pt_list, &parent->sync_pt_list_head); -+ return true; -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static void mali_internal_fence_value_str(struct dma_fence *fence, char *str, int size) -+#else -+static void mali_internal_fence_value_str(struct fence *fence, char *str, int size) -+#endif -+{ -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_timeline *parent; -+ -+ MALI_DEBUG_ASSERT_POINTER(fence); -+ MALI_IGNORE(str); -+ MALI_IGNORE(size); -+ -+ sync_pt = mali_internal_fence_to_sync_pt(fence); -+ parent = mali_internal_sync_pt_to_sync_timeline(sync_pt); -+ -+ parent->ops->print_sync_pt(sync_pt); -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+static const struct dma_fence_ops fence_ops = { -+#else -+static const struct fence_ops fence_ops = { -+#endif -+ .get_driver_name = mali_internal_fence_get_driver_name, -+ .get_timeline_name = mali_internal_fence_get_timeline_name, -+ .enable_signaling = mali_internal_fence_enable_signaling, -+ .signaled = mali_internal_fence_signaled, -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ .wait = dma_fence_default_wait, -+#else -+ .wait = fence_default_wait, -+#endif -+ .release = mali_internal_fence_release, -+ .fence_value_str = mali_internal_fence_value_str, -+}; -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h -new file mode 100755 -index 000000000..dbb29222b ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_internal_sync.h -@@ -0,0 +1,191 @@ -+/* -+ * Copyright (C) 2012-2015, 2017-2018 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_internal_sync.h -+ * -+ * Mali internal structure/interface for sync. -+ */ -+ -+#ifndef _MALI_INTERNAL_SYNC_H -+#define _MALI_INTERNAL_SYNC_H -+#include -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0) -+#include -+#include -+#include -+#include -+#include -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) -+#include -+#else -+#include -+#endif -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#include -+#else -+#include -+#endif -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+#include -+#else -+#include -+#endif -+#endif -+ -+struct mali_internal_sync_timeline; -+struct mali_internal_sync_point; -+struct mali_internal_sync_fence; -+ -+struct mali_internal_sync_timeline_ops { -+ const char *driver_name; -+ int (*has_signaled)(struct mali_internal_sync_point *pt); -+ void (*free_pt)(struct mali_internal_sync_point *sync_pt); -+ void (*release_obj)(struct mali_internal_sync_timeline *sync_timeline); -+ void (*print_sync_pt)(struct mali_internal_sync_point *sync_pt); -+}; -+ -+struct mali_internal_sync_timeline { -+ struct kref kref_count; -+ const struct mali_internal_sync_timeline_ops *ops; -+ char name[32]; -+ bool destroyed; -+ int fence_context; -+ int value; -+ spinlock_t sync_pt_list_lock; -+ struct list_head sync_pt_list_head; -+}; -+ -+struct mali_internal_sync_point { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ struct dma_fence base; -+#else -+ struct fence base; -+#endif -+ struct list_head sync_pt_list; -+}; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+struct mali_internal_sync_fence_cb { -+ struct fence_cb cb; -+ struct fence *fence; -+ struct mali_internal_sync_fence *sync_file; -+}; -+#endif -+ -+struct mali_internal_sync_fence { -+ struct file *file; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) -+ struct kref kref; -+#endif -+ char name[32]; -+#ifdef CONFIG_DEBUG_FS -+ struct list_head sync_file_list; -+#endif -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ int num_fences; -+#endif -+ wait_queue_head_t wq; -+#if LINUX_VERSION_CODE > KERNEL_VERSION(4, 12, 0) -+ unsigned long flags; -+#endif -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ atomic_t status; -+ struct mali_internal_sync_fence_cb cbs[]; -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ struct fence *fence; -+ struct fence_cb cb; -+#else -+ struct dma_fence *fence; -+ struct dma_fence_cb cb; -+#endif -+}; -+ -+struct mali_internal_sync_fence_waiter; -+ -+typedef void (*mali_internal_sync_callback_t)(struct mali_internal_sync_fence *sync_fence, -+ struct mali_internal_sync_fence_waiter *waiter); -+ -+struct mali_internal_sync_fence_waiter { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0) -+ wait_queue_entry_t work; -+#else -+ wait_queue_t work; -+#endif -+ mali_internal_sync_callback_t callback; -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ struct fence_cb cb; -+#else -+ struct dma_fence_cb cb; -+#endif -+#endif -+}; -+ -+/** -+ * Create a mali internal sync timeline. -+ * @param ops The implementation ops for the mali internal sync timeline -+ * @param size The size to allocate -+ * @param name The sync_timeline name -+ * @return The new mali internal sync timeline if successful, NULL if not. -+ */ -+struct mali_internal_sync_timeline *mali_internal_sync_timeline_create(const struct mali_internal_sync_timeline_ops *ops, -+ int size, const char *name); -+ -+/** -+ * Destroy one mali internal sync timeline. -+ * @param sync_timeline The mali internal sync timeline to destroy. -+ */ -+void mali_internal_sync_timeline_destroy(struct mali_internal_sync_timeline *sync_timeline); -+ -+/** -+ * Signal one mali internal sync timeline. -+ * @param sync_timeline The mali internal sync timeline to signal. -+ */ -+void mali_internal_sync_timeline_signal(struct mali_internal_sync_timeline *sync_timeline); -+ -+/** -+ * Create one mali internal sync point. -+ * @param sync_timeline The mali internal sync timeline to add this mali internal sync point. -+ * @return the new mali internal sync point if successful, NULL if not. -+ */ -+struct mali_internal_sync_point *mali_internal_sync_point_create(struct mali_internal_sync_timeline *sync_timeline, int size); -+ -+/** -+ * Merge mali internal sync fences -+ * @param sync_fence1 The mali internal sync fence to merge -+ * @param sync_fence2 The mali internal sync fence to merge -+ * @return the new mali internal sync fence if successful, NULL if not. -+ */ -+struct mali_internal_sync_fence *mali_internal_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, -+ struct mali_internal_sync_fence *sync_fence2); -+ -+/** -+ * Get the mali internal sync fence from sync fd -+ * @param fd The sync handle to get the mali internal sync fence -+ * @return the mali internal sync fence if successful, NULL if not. -+ */ -+struct mali_internal_sync_fence *mali_internal_sync_fence_fdget(int fd); -+ -+ -+void mali_internal_sync_fence_waiter_init(struct mali_internal_sync_fence_waiter *waiter, -+ mali_internal_sync_callback_t callback); -+ -+int mali_internal_sync_fence_wait_async(struct mali_internal_sync_fence *sync_fence, -+ struct mali_internal_sync_fence_waiter *waiter); -+ -+int mali_internal_sync_fence_cancel_async(struct mali_internal_sync_fence *sync_fence, -+ struct mali_internal_sync_fence_waiter *waiter); -+ -+#endif /*LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)*/ -+#endif /* _MALI_INTERNAL_SYNC_H */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c -new file mode 100755 -index 000000000..e45c7d2f2 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.c -@@ -0,0 +1,1154 @@ -+/** -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+ -+/** -+ * @file mali_kernel_linux.c -+ * Implementation of the Linux device driver entrypoints -+ */ -+#include "../platform/rk/custom_log.h" -+#include "../platform/rk/rk_ext.h" -+ -+#include /* kernel module definitions */ -+#include /* file system operations */ -+#include /* character device definitions */ -+#include /* memory manager definitions */ -+#include -+#include -+#include -+#include "mali_kernel_license.h" -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_kernel_core.h" -+#include "mali_osk.h" -+#include "mali_kernel_linux.h" -+#include "mali_ukk.h" -+#include "mali_ukk_wrappers.h" -+#include "mali_kernel_sysfs.h" -+#include "mali_pm.h" -+#include "mali_kernel_license.h" -+#include "mali_memory.h" -+#include "mali_memory_dma_buf.h" -+#include "mali_memory_manager.h" -+#include "mali_memory_swap_alloc.h" -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+#include "mali_profiling_internal.h" -+#endif -+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) -+#include "mali_osk_profiling.h" -+#include "mali_dvfs_policy.h" -+ -+static int is_first_resume = 1; -+/*Store the clk and vol for boot/insmod and mali_resume*/ -+static struct mali_gpu_clk_item mali_gpu_clk[2]; -+#endif -+ -+/* Streamline support for the Mali driver */ -+#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING) -+/* Ask Linux to create the tracepoints */ -+#define CREATE_TRACE_POINTS -+#include "mali_linux_trace.h" -+ -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters); -+#endif /* CONFIG_TRACEPOINTS */ -+ -+#ifdef CONFIG_MALI_DEVFREQ -+#include "mali_devfreq.h" -+#include "mali_osk_mali.h" -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) -+#include -+#else -+/* In 3.13 the OPP include header file, types, and functions were all -+ * renamed. Use the old filename for the include, and define the new names to -+ * the old, when an old kernel is detected. -+ */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) -+#include -+#else -+#include -+#endif /* Linux >= 3.13*/ -+#define dev_pm_opp_of_add_table of_init_opp_table -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) -+#define dev_pm_opp_of_remove_table of_free_opp_table -+#endif /* Linux >= 3.19 */ -+#endif /* Linux >= 4.4.0 */ -+#endif -+ -+/* from the __malidrv_build_info.c file that is generated during build */ -+extern const char *__malidrv_build_info(void); -+ -+/* Module parameter to control log level */ -+int mali_debug_level = 2; -+module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ -+MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output"); -+ -+extern int mali_max_job_runtime; -+module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what"); -+ -+extern int mali_l2_max_reads; -+module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache"); -+ -+extern unsigned int mali_dedicated_mem_start; -+module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory."); -+ -+extern unsigned int mali_dedicated_mem_size; -+module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory."); -+ -+extern unsigned int mali_shared_mem_size; -+module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory."); -+ -+#if defined(CONFIG_MALI400_PROFILING) -+extern int mali_boot_profiling; -+module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization"); -+#endif -+ -+extern int mali_max_pp_cores_group_1; -+module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group."); -+ -+extern int mali_max_pp_cores_group_2; -+module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only)."); -+ -+extern unsigned int mali_mem_swap_out_threshold_value; -+module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver."); -+ -+#if defined(CONFIG_MALI_DVFS) -+/** the max fps the same as display vsync default 60, can set by module insert parameter */ -+extern int mali_max_system_fps; -+module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC."); -+ -+/** a lower limit on their desired FPS default 58, can set by module insert parameter*/ -+extern int mali_desired_fps; -+module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); -+MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps"); -+#endif -+ -+#if MALI_ENABLE_CPU_CYCLES -+#include -+#include -+#include -+static struct timer_list mali_init_cpu_clock_timers[8]; -+static u32 mali_cpu_clock_last_value[8] = {0,}; -+#endif -+ -+/* Export symbols from common code: mali_user_settings.c */ -+#include "mali_user_settings_db.h" -+EXPORT_SYMBOL(mali_set_user_setting); -+EXPORT_SYMBOL(mali_get_user_setting); -+ -+static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */ -+ -+/* This driver only supports one Mali device, and this variable stores this single platform device */ -+struct platform_device *mali_platform_device = NULL; -+ -+/* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */ -+static struct miscdevice mali_miscdevice = { 0, }; -+ -+static int mali_miscdevice_register(struct platform_device *pdev); -+static void mali_miscdevice_unregister(void); -+ -+static int mali_open(struct inode *inode, struct file *filp); -+static int mali_release(struct inode *inode, struct file *filp); -+static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); -+ -+static int mali_probe(struct platform_device *pdev); -+static int mali_remove(struct platform_device *pdev); -+ -+static int mali_driver_suspend_scheduler(struct device *dev); -+static int mali_driver_resume_scheduler(struct device *dev); -+ -+#ifdef CONFIG_PM_RUNTIME -+static int mali_driver_runtime_suspend(struct device *dev); -+static int mali_driver_runtime_resume(struct device *dev); -+static int mali_driver_runtime_idle(struct device *dev); -+#endif -+ -+#if defined(MALI_FAKE_PLATFORM_DEVICE) -+#if defined(CONFIG_MALI_DT) -+extern int mali_platform_device_init(struct platform_device *device); -+extern int mali_platform_device_deinit(struct platform_device *device); -+#else -+extern int mali_platform_device_register(void); -+extern int mali_platform_device_unregister(void); -+#endif -+#endif -+ -+extern int rk_platform_init_opp_table(struct device *dev); -+ -+/* Linux power management operations provided by the Mali device driver */ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) -+struct pm_ext_ops mali_dev_ext_pm_ops = { -+ .base = -+ { -+ .suspend = mali_driver_suspend_scheduler, -+ .resume = mali_driver_resume_scheduler, -+ .freeze = mali_driver_suspend_scheduler, -+ .thaw = mali_driver_resume_scheduler, -+ }, -+}; -+#else -+static const struct dev_pm_ops mali_dev_pm_ops = { -+#ifdef CONFIG_PM_RUNTIME -+ .runtime_suspend = mali_driver_runtime_suspend, -+ .runtime_resume = mali_driver_runtime_resume, -+ .runtime_idle = mali_driver_runtime_idle, -+#endif -+ .suspend = mali_driver_suspend_scheduler, -+ .resume = mali_driver_resume_scheduler, -+ .freeze = mali_driver_suspend_scheduler, -+ .thaw = mali_driver_resume_scheduler, -+ .poweroff = mali_driver_suspend_scheduler, -+}; -+#endif -+ -+#ifdef CONFIG_MALI_DT -+static struct of_device_id base_dt_ids[] = { -+ {.compatible = "arm,mali-300"}, -+ /*-------------------------------------------------------*/ -+ /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ -+ // {.compatible = "arm,mali-400"}, -+ {.compatible = "arm,mali400"}, -+ /*-------------------------------------------------------*/ -+ {.compatible = "arm,mali-450"}, -+ {.compatible = "arm,mali-470"}, -+ {}, -+}; -+ -+MODULE_DEVICE_TABLE(of, base_dt_ids); -+#endif -+ -+/* The Mali device driver struct */ -+static struct platform_driver mali_platform_driver = { -+ .probe = mali_probe, -+ .remove = mali_remove, -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)) -+ .pm = &mali_dev_ext_pm_ops, -+#endif -+ .driver = -+ { -+ .name = MALI_GPU_NAME_UTGARD, -+ .owner = THIS_MODULE, -+ .bus = &platform_bus_type, -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) -+ .pm = &mali_dev_pm_ops, -+#endif -+#ifdef CONFIG_MALI_DT -+ .of_match_table = of_match_ptr(base_dt_ids), -+#endif -+ }, -+}; -+ -+/* Linux misc device operations (/dev/mali) */ -+struct file_operations mali_fops = { -+ .owner = THIS_MODULE, -+ .open = mali_open, -+ .release = mali_release, -+ .unlocked_ioctl = mali_ioctl, -+ .compat_ioctl = mali_ioctl, -+ .mmap = mali_mmap -+}; -+ -+#if MALI_ENABLE_CPU_CYCLES -+void mali_init_cpu_time_counters(int reset, int enable_divide_by_64) -+{ -+ /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */ -+ u32 write_value; -+ -+ /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */ -+ /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */ -+ asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f)); -+ -+ -+ /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */ -+ write_value = 1 << 0; /* Bit 0 set. Enable counters */ -+ if (reset) { -+ write_value |= 1 << 1; /* Reset event counters */ -+ write_value |= 1 << 2; /* Reset cycle counter */ -+ } -+ if (enable_divide_by_64) { -+ write_value |= 1 << 3; /* Enable the Clock divider by 64 */ -+ } -+ write_value |= 1 << 4; /* Export enable. Not needed */ -+ asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value)); -+ -+ /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */ -+ asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f)); -+ -+ -+ /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */ -+ /* User mode access to the Performance Monitors enabled. */ -+ /* Lets User space read cpu clock cycles */ -+ asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1)); -+} -+ -+/** A timer function that configures the cycle clock counter on current CPU. -+ * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this -+ * function to trigger on all Cpus during module load. -+ */ -+static void mali_init_cpu_clock_timer_func(unsigned long data) -+{ -+ int reset_counters, enable_divide_clock_counter_by_64; -+ int current_cpu = raw_smp_processor_id(); -+ unsigned int sample0; -+ unsigned int sample1; -+ -+ MALI_IGNORE(data); -+ -+ reset_counters = 1; -+ enable_divide_clock_counter_by_64 = 0; -+ mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64); -+ -+ sample0 = mali_get_cpu_cyclecount(); -+ sample1 = mali_get_cpu_cyclecount(); -+ -+ MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1)); -+} -+ -+/** A timer functions for storing current time on all cpus. -+ * Used for checking if the clocks have similar values or if they are drifting. -+ */ -+static void mali_print_cpu_clock_timer_func(unsigned long data) -+{ -+ int current_cpu = raw_smp_processor_id(); -+ unsigned int sample0; -+ -+ MALI_IGNORE(data); -+ sample0 = mali_get_cpu_cyclecount(); -+ if (current_cpu < 8) { -+ mali_cpu_clock_last_value[current_cpu] = sample0; -+ } -+} -+ -+/** Init the performance registers on all CPUs to count clock cycles. -+ * For init \a print_only should be 0. -+ * If \a print_only is 1, it will intead print the current clock value of all CPUs. -+ */ -+void mali_init_cpu_time_counters_on_all_cpus(int print_only) -+{ -+ int i = 0; -+ int cpu_number; -+ int jiffies_trigger; -+ int jiffies_wait; -+ -+ jiffies_wait = 2; -+ jiffies_trigger = jiffies + jiffies_wait; -+ -+ for (i = 0 ; i < 8 ; i++) { -+ init_timer(&mali_init_cpu_clock_timers[i]); -+ if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func; -+ else mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func; -+ mali_init_cpu_clock_timers[i].expires = jiffies_trigger ; -+ } -+ cpu_number = cpumask_first(cpu_online_mask); -+ for (i = 0 ; i < 8 ; i++) { -+ int next_cpu; -+ add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number); -+ next_cpu = cpumask_next(cpu_number, cpu_online_mask); -+ if (next_cpu >= nr_cpu_ids) break; -+ cpu_number = next_cpu; -+ } -+ -+ while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait); -+ -+ for (i = 0 ; i < 8 ; i++) { -+ del_timer_sync(&mali_init_cpu_clock_timers[i]); -+ } -+ -+ if (print_only) { -+ if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) { -+ /* Diff can be printed if we want to check if the clocks are in sync -+ int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/ -+ MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1])); -+ } else { -+ MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3])); -+ } -+ } -+} -+#endif -+ -+int mali_module_init(void) -+{ -+ int err = 0; -+ -+ MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION)); -+ MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__)); -+ MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING)); -+ -+ I("svn_rev_string_from_arm of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.", -+ SVN_REV_STRING, -+ RK_KO_VER, -+ __TIME__, -+ __DATE__); -+ -+#if MALI_ENABLE_CPU_CYCLES -+ mali_init_cpu_time_counters_on_all_cpus(0); -+ MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n")); -+ /* Printing the current cpu counters */ -+ mali_init_cpu_time_counters_on_all_cpus(1); -+#endif -+ -+ /* Initialize module wide settings */ -+#ifdef MALI_FAKE_PLATFORM_DEVICE -+#ifndef CONFIG_MALI_DT -+ MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n")); -+ err = mali_platform_device_register(); -+ if (0 != err) { -+ return err; -+ } -+#endif -+#endif -+ -+ MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n")); -+ -+ err = platform_driver_register(&mali_platform_driver); -+ -+ if (0 != err) { -+ MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err)); -+#ifdef MALI_FAKE_PLATFORM_DEVICE -+#ifndef CONFIG_MALI_DT -+ mali_platform_device_unregister(); -+#endif -+#endif -+ mali_platform_device = NULL; -+ return err; -+ } -+ -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+ err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE); -+ if (0 != err) { -+ /* No biggie if we wheren't able to initialize the profiling */ -+ MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n")); -+ } -+#endif -+ -+ /* Tracing the current frequency and voltage from boot/insmod*/ -+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) -+ /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/ -+ mali_get_current_gpu_clk_item(&mali_gpu_clk[0]); -+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | -+ MALI_PROFILING_EVENT_CHANNEL_GPU | -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, -+ mali_gpu_clk[0].clock, -+ mali_gpu_clk[0].vol / 1000, -+ 0, 0, 0); -+#endif -+ -+ MALI_PRINT(("Mali device driver loaded\n")); -+ -+ return 0; /* Success */ -+} -+ -+void mali_module_exit(void) -+{ -+ MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION)); -+ -+ MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n")); -+ -+ platform_driver_unregister(&mali_platform_driver); -+ -+#if defined(MALI_FAKE_PLATFORM_DEVICE) -+#ifndef CONFIG_MALI_DT -+ MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n")); -+ mali_platform_device_unregister(); -+#endif -+#endif -+ -+ /* Tracing the current frequency and voltage from rmmod*/ -+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | -+ MALI_PROFILING_EVENT_CHANNEL_GPU | -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, -+ 0, -+ 0, -+ 0, 0, 0); -+ -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+ _mali_internal_profiling_term(); -+#endif -+ -+ MALI_PRINT(("Mali device driver unloaded\n")); -+} -+ -+#ifdef CONFIG_MALI_DEVFREQ -+struct mali_device *mali_device_alloc(void) -+{ -+ return kzalloc(sizeof(struct mali_device), GFP_KERNEL); -+} -+ -+void mali_device_free(struct mali_device *mdev) -+{ -+ kfree(mdev); -+} -+#endif -+ -+static int mali_probe(struct platform_device *pdev) -+{ -+ int err; -+#ifdef CONFIG_MALI_DEVFREQ -+ struct mali_device *mdev; -+#endif -+ -+ MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name)); -+ -+ if (NULL != mali_platform_device) { -+ /* Already connected to a device, return error */ -+ MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device.")); -+ return -EEXIST; -+ } -+ -+ mali_platform_device = pdev; -+ -+ dev_info(&pdev->dev, "mali_platform_device->num_resources = %d\n", -+ mali_platform_device->num_resources); -+ -+ { -+ int i = 0; -+ -+ for(i = 0; i < mali_platform_device->num_resources; i++) -+ dev_info(&pdev->dev, -+ "resource[%d].start = 0x%pa\n", -+ i, -+ &mali_platform_device->resource[i].start); -+ } -+ -+#ifdef CONFIG_MALI_DT -+ /* If we use DT to initialize our DDK, we have to prepare somethings. */ -+ err = mali_platform_device_init(mali_platform_device); -+ if (0 != err) { -+ MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device.")); -+ mali_platform_device = NULL; -+ return -EFAULT; -+ } -+#endif -+ -+#ifdef CONFIG_MALI_DEVFREQ -+ mdev = mali_device_alloc(); -+ if (!mdev) { -+ MALI_PRINT_ERROR(("Can't allocate mali device private data\n")); -+ return -ENOMEM; -+ } -+ -+ mdev->dev = &pdev->dev; -+ dev_set_drvdata(mdev->dev, mdev); -+ -+ /*Initilization clock and regulator*/ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_REGULATOR) -+ mdev->regulator = regulator_get_optional(mdev->dev, "mali"); -+ if (IS_ERR_OR_NULL(mdev->regulator)) { -+ MALI_DEBUG_PRINT(2, ("Continuing without Mali regulator control\n")); -+ mdev->regulator = NULL; -+ /* Allow probe to continue without regulator */ -+ } -+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ -+ -+ err = rk_platform_init_opp_table(mdev->dev); -+ if (err) -+ MALI_DEBUG_PRINT(3, ("Failed to init_opp_table\n")); -+ -+ /* Need to name the gpu clock "clk_mali" in the device tree */ -+ mdev->clock = clk_get(mdev->dev, "clk_mali"); -+ if (IS_ERR_OR_NULL(mdev->clock)) { -+ MALI_DEBUG_PRINT(2, ("Continuing without Mali clock control\n")); -+ mdev->clock = NULL; -+ /* Allow probe to continue without clock. */ -+ } else { -+ err = clk_prepare(mdev->clock); -+ if (err) { -+ MALI_PRINT_ERROR(("Failed to prepare clock (%d)\n", err)); -+ goto clock_prepare_failed; -+ } -+ } -+ -+ /* initilize pm metrics related */ -+ if (mali_pm_metrics_init(mdev) < 0) { -+ MALI_DEBUG_PRINT(2, ("mali pm metrics init failed\n")); -+ goto pm_metrics_init_failed; -+ } -+ -+ if (mali_devfreq_init(mdev) < 0) { -+ MALI_DEBUG_PRINT(2, ("mali devfreq init failed\n")); -+ goto devfreq_init_failed; -+ } -+#endif -+ -+ -+ if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) { -+ /* Initialize the Mali GPU HW specified by pdev */ -+ if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) { -+ /* Register a misc device (so we are accessible from user space) */ -+ err = mali_miscdevice_register(pdev); -+ if (0 == err) { -+ /* Setup sysfs entries */ -+ err = mali_sysfs_register(mali_dev_name); -+ -+ if (0 == err) { -+ MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name)); -+ -+ return 0; -+ } else { -+ MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries")); -+ } -+ mali_miscdevice_unregister(); -+ } else { -+ MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device.")); -+ } -+ mali_terminate_subsystems(); -+ } else { -+ MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver.")); -+ } -+ _mali_osk_wq_term(); -+ } -+ -+#ifdef CONFIG_MALI_DEVFREQ -+ mali_devfreq_term(mdev); -+devfreq_init_failed: -+ mali_pm_metrics_term(mdev); -+pm_metrics_init_failed: -+ clk_unprepare(mdev->clock); -+clock_prepare_failed: -+ clk_put(mdev->clock); -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_PM_OPP) -+ dev_pm_opp_of_remove_table(mdev->dev); -+#endif -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_REGULATOR) -+ regulator_put(mdev->regulator); -+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ -+ mali_device_free(mdev); -+#endif -+ -+#ifdef CONFIG_MALI_DT -+ mali_platform_device_deinit(mali_platform_device); -+#endif -+ mali_platform_device = NULL; -+ return -EFAULT; -+} -+ -+static int mali_remove(struct platform_device *pdev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ struct mali_device *mdev = dev_get_drvdata(&pdev->dev); -+#endif -+ -+ MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name)); -+ mali_sysfs_unregister(); -+ mali_miscdevice_unregister(); -+ mali_terminate_subsystems(); -+ _mali_osk_wq_term(); -+ -+#ifdef CONFIG_MALI_DEVFREQ -+ mali_devfreq_term(mdev); -+ -+ mali_pm_metrics_term(mdev); -+ -+ if (mdev->clock) { -+ clk_unprepare(mdev->clock); -+ clk_put(mdev->clock); -+ mdev->clock = NULL; -+ } -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_PM_OPP) -+ dev_pm_opp_of_remove_table(mdev->dev); -+#endif -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_REGULATOR) -+ regulator_put(mdev->regulator); -+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ -+ mali_device_free(mdev); -+#endif -+ -+#ifdef CONFIG_MALI_DT -+ mali_platform_device_deinit(mali_platform_device); -+#endif -+ mali_platform_device = NULL; -+ return 0; -+} -+ -+static int mali_miscdevice_register(struct platform_device *pdev) -+{ -+ int err; -+ -+ mali_miscdevice.minor = MISC_DYNAMIC_MINOR; -+ mali_miscdevice.name = mali_dev_name; -+ mali_miscdevice.fops = &mali_fops; -+ mali_miscdevice.parent = get_device(&pdev->dev); -+ -+ err = misc_register(&mali_miscdevice); -+ if (0 != err) { -+ MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err)); -+ } -+ -+ return err; -+} -+ -+static void mali_miscdevice_unregister(void) -+{ -+ misc_deregister(&mali_miscdevice); -+} -+ -+static int mali_driver_suspend_scheduler(struct device *dev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ if (!mdev) -+ return -ENODEV; -+#endif -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ devfreq_suspend_device(mdev->devfreq); -+#endif -+ -+ mali_pm_os_suspend(MALI_TRUE); -+ /* Tracing the frequency and voltage after mali is suspended */ -+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | -+ MALI_PROFILING_EVENT_CHANNEL_GPU | -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, -+ 0, -+ 0, -+ 0, 0, 0); -+ return 0; -+} -+ -+static int mali_driver_resume_scheduler(struct device *dev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ if (!mdev) -+ return -ENODEV; -+#endif -+ -+ /* Tracing the frequency and voltage after mali is resumed */ -+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) -+ /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ -+ if (is_first_resume == 1) { -+ mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); -+ is_first_resume = 0; -+ } -+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | -+ MALI_PROFILING_EVENT_CHANNEL_GPU | -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, -+ mali_gpu_clk[1].clock, -+ mali_gpu_clk[1].vol / 1000, -+ 0, 0, 0); -+#endif -+ mali_pm_os_resume(); -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ devfreq_resume_device(mdev->devfreq); -+#endif -+ -+ return 0; -+} -+ -+#ifdef CONFIG_PM_RUNTIME -+static int mali_driver_runtime_suspend(struct device *dev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ if (!mdev) -+ return -ENODEV; -+#endif -+ -+ if (MALI_TRUE == mali_pm_runtime_suspend()) { -+ /* Tracing the frequency and voltage after mali is suspended */ -+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | -+ MALI_PROFILING_EVENT_CHANNEL_GPU | -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, -+ 0, -+ 0, -+ 0, 0, 0); -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ MALI_DEBUG_PRINT(4, ("devfreq_suspend_device: stop devfreq monitor\n")); -+ devfreq_suspend_device(mdev->devfreq); -+#endif -+ -+ return 0; -+ } else { -+ return -EBUSY; -+ } -+} -+ -+static int mali_driver_runtime_resume(struct device *dev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ if (!mdev) -+ return -ENODEV; -+#endif -+ -+ /* Tracing the frequency and voltage after mali is resumed */ -+#if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS) -+ /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/ -+ if (is_first_resume == 1) { -+ mali_get_current_gpu_clk_item(&mali_gpu_clk[1]); -+ is_first_resume = 0; -+ } -+ _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | -+ MALI_PROFILING_EVENT_CHANNEL_GPU | -+ MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE, -+ mali_gpu_clk[1].clock, -+ mali_gpu_clk[1].vol / 1000, -+ 0, 0, 0); -+#endif -+ -+ mali_pm_runtime_resume(); -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ MALI_DEBUG_PRINT(4, ("devfreq_resume_device: start devfreq monitor\n")); -+ devfreq_resume_device(mdev->devfreq); -+#endif -+ return 0; -+} -+ -+static int mali_driver_runtime_idle(struct device *dev) -+{ -+ /* Nothing to do */ -+ return 0; -+} -+#endif -+ -+static int mali_open(struct inode *inode, struct file *filp) -+{ -+ struct mali_session_data *session_data; -+ _mali_osk_errcode_t err; -+ -+ /* input validation */ -+ if (mali_miscdevice.minor != iminor(inode)) { -+ MALI_PRINT_ERROR(("mali_open() Minor does not match\n")); -+ return -ENODEV; -+ } -+ -+ /* allocated struct to track this session */ -+ err = _mali_ukk_open((void **)&session_data); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ /* initialize file pointer */ -+ filp->f_pos = 0; -+ -+ /* link in our session data */ -+ filp->private_data = (void *)session_data; -+ -+ filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping; -+ -+ return 0; -+} -+ -+static int mali_release(struct inode *inode, struct file *filp) -+{ -+ _mali_osk_errcode_t err; -+ -+ /* input validation */ -+ if (mali_miscdevice.minor != iminor(inode)) { -+ MALI_PRINT_ERROR(("mali_release() Minor does not match\n")); -+ return -ENODEV; -+ } -+ -+ err = _mali_ukk_close((void **)&filp->private_data); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ return 0; -+} -+ -+int map_errcode(_mali_osk_errcode_t err) -+{ -+ switch (err) { -+ case _MALI_OSK_ERR_OK : -+ return 0; -+ case _MALI_OSK_ERR_FAULT: -+ return -EFAULT; -+ case _MALI_OSK_ERR_INVALID_FUNC: -+ return -ENOTTY; -+ case _MALI_OSK_ERR_INVALID_ARGS: -+ return -EINVAL; -+ case _MALI_OSK_ERR_NOMEM: -+ return -ENOMEM; -+ case _MALI_OSK_ERR_TIMEOUT: -+ return -ETIMEDOUT; -+ case _MALI_OSK_ERR_RESTARTSYSCALL: -+ return -ERESTARTSYS; -+ case _MALI_OSK_ERR_ITEM_NOT_FOUND: -+ return -ENOENT; -+ default: -+ return -EFAULT; -+ } -+} -+ -+static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -+{ -+ int err; -+ struct mali_session_data *session_data; -+ -+ MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg)); -+ -+ session_data = (struct mali_session_data *)filp->private_data; -+ if (NULL == session_data) { -+ MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n")); -+ return -ENOTTY; -+ } -+ -+ if (NULL == (void *)arg) { -+ MALI_DEBUG_PRINT(7, ("arg was NULL\n")); -+ return -ENOTTY; -+ } -+ -+ switch (cmd) { -+ case MALI_IOC_WAIT_FOR_NOTIFICATION: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64))); -+ err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg); -+ break; -+ -+ case MALI_IOC_GET_API_VERSION_V2: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64))); -+ err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg); -+ break; -+ -+ case MALI_IOC_GET_API_VERSION: -+ err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg); -+ break; -+ -+ case MALI_IOC_POST_NOTIFICATION: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64))); -+ err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg); -+ break; -+ -+ /* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ -+#if 0 -+ case MALI_IOC_GET_MALI_VERSION_IN_RK30: -+ err = get_mali_version_in_rk30_wrapper(session_data, (_mali_uk_get_mali_version_in_rk30_s __user *)arg); -+ break; -+#else -+ case MALI_IOC_GET_RK_KO_VERSION: -+ err = get_rk_ko_version_wrapper(session_data, (_mali_rk_ko_version_s __user *)arg); -+ break; -+#endif -+ -+ case MALI_IOC_GET_USER_SETTINGS: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64))); -+ err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg); -+ break; -+ -+ case MALI_IOC_REQUEST_HIGH_PRIORITY: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64))); -+ err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PENDING_SUBMIT: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64))); -+ err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg); -+ break; -+ -+#if defined(CONFIG_MALI400_PROFILING) -+ case MALI_IOC_PROFILING_ADD_EVENT: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64))); -+ err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64))); -+ err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PROFILING_STREAM_FD_GET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64))); -+ err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PROILING_CONTROL_SET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64))); -+ err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg); -+ break; -+#else -+ -+ case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */ -+ case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */ -+ MALI_DEBUG_PRINT(2, ("Profiling not supported\n")); -+ err = -ENOTTY; -+ break; -+#endif -+ -+ case MALI_IOC_PROFILING_MEMORY_USAGE_GET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64))); -+ err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_ALLOC: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64))); -+ err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_FREE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64))); -+ err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_BIND: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64))); -+ err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_UNBIND: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64))); -+ err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_COW: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64))); -+ err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_COW_MODIFY_RANGE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64))); -+ err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_RESIZE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64))); -+ err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_WRITE_SAFE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64))); -+ err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64))); -+ err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64))); -+ err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg); -+ break; -+ -+ case MALI_IOC_MEM_DMA_BUF_GET_SIZE: -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64))); -+ err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg); -+#else -+ MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n")); -+ err = -ENOTTY; -+#endif -+ break; -+ -+ case MALI_IOC_PP_START_JOB: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64))); -+ err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PP_AND_GP_START_JOB: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64))); -+ err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PP_NUMBER_OF_CORES_GET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64))); -+ err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PP_CORE_VERSION_GET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64))); -+ err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg); -+ break; -+ -+ case MALI_IOC_PP_DISABLE_WB: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64))); -+ err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg); -+ break; -+ -+ case MALI_IOC_GP2_START_JOB: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64))); -+ err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg); -+ break; -+ -+ case MALI_IOC_GP2_NUMBER_OF_CORES_GET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64))); -+ err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg); -+ break; -+ -+ case MALI_IOC_GP2_CORE_VERSION_GET: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64))); -+ err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg); -+ break; -+ -+ case MALI_IOC_GP2_SUSPEND_RESPONSE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64))); -+ err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg); -+ break; -+ -+ case MALI_IOC_VSYNC_EVENT_REPORT: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64))); -+ err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg); -+ break; -+ -+ case MALI_IOC_TIMELINE_GET_LATEST_POINT: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64))); -+ err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg); -+ break; -+ case MALI_IOC_TIMELINE_WAIT: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64))); -+ err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg); -+ break; -+ case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64))); -+ err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg); -+ break; -+ case MALI_IOC_SOFT_JOB_START: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64))); -+ err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg); -+ break; -+ case MALI_IOC_SOFT_JOB_SIGNAL: -+ BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64))); -+ err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg); -+ break; -+ -+ default: -+ MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg)); -+ err = -ENOTTY; -+ }; -+ -+ return err; -+} -+ -+late_initcall_sync(mali_module_init); -+module_exit(mali_module_exit); -+ -+MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE); -+MODULE_AUTHOR("ARM Ltd."); -+MODULE_VERSION(SVN_REV_STRING); -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h -new file mode 100755 -index 000000000..be754cb15 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_linux.h -@@ -0,0 +1,36 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_KERNEL_LINUX_H__ -+#define __MALI_KERNEL_LINUX_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include /* character device definitions */ -+#include -+#include -+#include "mali_kernel_license.h" -+#include "mali_osk_types.h" -+#include -+ -+extern struct platform_device *mali_platform_device; -+ -+/* After 3.19.0 kenrel droped CONFIG_PM_RUNTIME define,define by ourself */ -+#if defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) -+#define CONFIG_PM_RUNTIME 1 -+#endif -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_KERNEL_LINUX_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c -new file mode 100755 -index 000000000..7bda438fe ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.c -@@ -0,0 +1,1410 @@ -+/** -+ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+ -+/** -+ * @file mali_kernel_sysfs.c -+ * Implementation of some sysfs data exports -+ */ -+ -+#include -+#include -+#include -+#include -+#include "mali_kernel_license.h" -+#include "mali_kernel_common.h" -+#include "mali_ukk.h" -+ -+#if MALI_LICENSE_IS_GPL -+ -+#include -+#include -+#include -+#include -+#include -+#include "mali_kernel_sysfs.h" -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+#include -+#include "mali_osk_profiling.h" -+#endif -+ -+#include -+#include "mali_pm.h" -+#include "mali_pmu.h" -+#include "mali_group.h" -+#include "mali_gp.h" -+#include "mali_pp.h" -+#include "mali_l2_cache.h" -+#include "mali_hw_core.h" -+#include "mali_kernel_core.h" -+#include "mali_user_settings_db.h" -+#include "mali_profiling_internal.h" -+#include "mali_gp_job.h" -+#include "mali_pp_job.h" -+#include "mali_executor.h" -+ -+#define PRIVATE_DATA_COUNTER_MAKE_GP(src) (src) -+#define PRIVATE_DATA_COUNTER_MAKE_PP(src) ((1 << 24) | src) -+#define PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(src, sub_job) ((1 << 24) | (1 << 16) | (sub_job << 8) | src) -+#define PRIVATE_DATA_COUNTER_IS_PP(a) ((((a) >> 24) & 0xFF) ? MALI_TRUE : MALI_FALSE) -+#define PRIVATE_DATA_COUNTER_GET_SRC(a) (a & 0xFF) -+#define PRIVATE_DATA_COUNTER_IS_SUB_JOB(a) ((((a) >> 16) & 0xFF) ? MALI_TRUE : MALI_FALSE) -+#define PRIVATE_DATA_COUNTER_GET_SUB_JOB(a) (((a) >> 8) & 0xFF) -+ -+#define POWER_BUFFER_SIZE 3 -+ -+static struct dentry *mali_debugfs_dir = NULL; -+ -+typedef enum { -+ _MALI_DEVICE_SUSPEND, -+ _MALI_DEVICE_RESUME, -+ _MALI_DEVICE_DVFS_PAUSE, -+ _MALI_DEVICE_DVFS_RESUME, -+ _MALI_MAX_EVENTS -+} _mali_device_debug_power_events; -+ -+static const char *const mali_power_events[_MALI_MAX_EVENTS] = { -+ [_MALI_DEVICE_SUSPEND] = "suspend", -+ [_MALI_DEVICE_RESUME] = "resume", -+ [_MALI_DEVICE_DVFS_PAUSE] = "dvfs_pause", -+ [_MALI_DEVICE_DVFS_RESUME] = "dvfs_resume", -+}; -+ -+static mali_bool power_always_on_enabled = MALI_FALSE; -+ -+static int open_copy_private_data(struct inode *inode, struct file *filp) -+{ -+ filp->private_data = inode->i_private; -+ return 0; -+} -+ -+static ssize_t group_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) -+{ -+ int r; -+ char buffer[64]; -+ struct mali_group *group; -+ -+ group = (struct mali_group *)filp->private_data; -+ MALI_DEBUG_ASSERT_POINTER(group); -+ -+ r = snprintf(buffer, 64, "%u\n", -+ mali_executor_group_is_disabled(group) ? 0 : 1); -+ -+ return simple_read_from_buffer(buf, count, offp, buffer, r); -+} -+ -+static ssize_t group_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) -+{ -+ int r; -+ char buffer[64]; -+ unsigned long val; -+ struct mali_group *group; -+ -+ group = (struct mali_group *)filp->private_data; -+ MALI_DEBUG_ASSERT_POINTER(group); -+ -+ if (count >= sizeof(buffer)) { -+ return -ENOMEM; -+ } -+ -+ if (copy_from_user(&buffer[0], buf, count)) { -+ return -EFAULT; -+ } -+ buffer[count] = '\0'; -+ -+ r = kstrtoul(&buffer[0], 10, &val); -+ if (0 != r) { -+ return -EINVAL; -+ } -+ -+ switch (val) { -+ case 1: -+ mali_executor_group_enable(group); -+ break; -+ case 0: -+ mali_executor_group_disable(group); -+ break; -+ default: -+ return -EINVAL; -+ break; -+ } -+ -+ *offp += count; -+ return count; -+} -+ -+static const struct file_operations group_enabled_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = group_enabled_read, -+ .write = group_enabled_write, -+}; -+ -+static ssize_t hw_core_base_addr_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) -+{ -+ int r; -+ char buffer[64]; -+ struct mali_hw_core *hw_core; -+ -+ hw_core = (struct mali_hw_core *)filp->private_data; -+ MALI_DEBUG_ASSERT_POINTER(hw_core); -+ -+ r = snprintf(buffer, 64, "0x%lX\n", hw_core->phys_addr); -+ -+ return simple_read_from_buffer(buf, count, offp, buffer, r); -+} -+ -+static const struct file_operations hw_core_base_addr_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = hw_core_base_addr_read, -+}; -+ -+static ssize_t profiling_counter_src_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); -+ u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); -+ mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); -+ u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); -+ char buf[64]; -+ int r; -+ u32 val; -+ -+ if (MALI_TRUE == is_pp) { -+ /* PP counter */ -+ if (MALI_TRUE == is_sub_job) { -+ /* Get counter for a particular sub job */ -+ if (0 == src_id) { -+ val = mali_pp_job_get_pp_counter_sub_job_src0(sub_job); -+ } else { -+ val = mali_pp_job_get_pp_counter_sub_job_src1(sub_job); -+ } -+ } else { -+ /* Get default counter for all PP sub jobs */ -+ if (0 == src_id) { -+ val = mali_pp_job_get_pp_counter_global_src0(); -+ } else { -+ val = mali_pp_job_get_pp_counter_global_src1(); -+ } -+ } -+ } else { -+ /* GP counter */ -+ if (0 == src_id) { -+ val = mali_gp_job_get_gp_counter_src0(); -+ } else { -+ val = mali_gp_job_get_gp_counter_src1(); -+ } -+ } -+ -+ if (MALI_HW_CORE_NO_COUNTER == val) { -+ r = snprintf(buf, 64, "-1\n"); -+ } else { -+ r = snprintf(buf, 64, "%u\n", val); -+ } -+ -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t profiling_counter_src_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ u32 is_pp = PRIVATE_DATA_COUNTER_IS_PP((uintptr_t)filp->private_data); -+ u32 src_id = PRIVATE_DATA_COUNTER_GET_SRC((uintptr_t)filp->private_data); -+ mali_bool is_sub_job = PRIVATE_DATA_COUNTER_IS_SUB_JOB((uintptr_t)filp->private_data); -+ u32 sub_job = PRIVATE_DATA_COUNTER_GET_SUB_JOB((uintptr_t)filp->private_data); -+ char buf[64]; -+ long val; -+ int ret; -+ -+ if (cnt >= sizeof(buf)) { -+ return -EINVAL; -+ } -+ -+ if (copy_from_user(&buf, ubuf, cnt)) { -+ return -EFAULT; -+ } -+ -+ buf[cnt] = 0; -+ -+ ret = kstrtol(buf, 10, &val); -+ if (ret < 0) { -+ return ret; -+ } -+ -+ if (val < 0) { -+ /* any negative input will disable counter */ -+ val = MALI_HW_CORE_NO_COUNTER; -+ } -+ -+ if (MALI_TRUE == is_pp) { -+ /* PP counter */ -+ if (MALI_TRUE == is_sub_job) { -+ /* Set counter for a particular sub job */ -+ if (0 == src_id) { -+ mali_pp_job_set_pp_counter_sub_job_src0(sub_job, (u32)val); -+ } else { -+ mali_pp_job_set_pp_counter_sub_job_src1(sub_job, (u32)val); -+ } -+ } else { -+ /* Set default counter for all PP sub jobs */ -+ if (0 == src_id) { -+ mali_pp_job_set_pp_counter_global_src0((u32)val); -+ } else { -+ mali_pp_job_set_pp_counter_global_src1((u32)val); -+ } -+ } -+ } else { -+ /* GP counter */ -+ if (0 == src_id) { -+ mali_gp_job_set_gp_counter_src0((u32)val); -+ } else { -+ mali_gp_job_set_gp_counter_src1((u32)val); -+ } -+ } -+ -+ *ppos += cnt; -+ return cnt; -+} -+ -+static const struct file_operations profiling_counter_src_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = profiling_counter_src_read, -+ .write = profiling_counter_src_write, -+}; -+ -+static ssize_t l2_l2x_counter_srcx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) -+{ -+ char buf[64]; -+ int r; -+ u32 val; -+ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; -+ -+ if (0 == src_id) { -+ val = mali_l2_cache_core_get_counter_src0(l2_core); -+ } else { -+ val = mali_l2_cache_core_get_counter_src1(l2_core); -+ } -+ -+ if (MALI_HW_CORE_NO_COUNTER == val) { -+ r = snprintf(buf, 64, "-1\n"); -+ } else { -+ r = snprintf(buf, 64, "%u\n", val); -+ } -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t l2_l2x_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) -+{ -+ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; -+ char buf[64]; -+ long val; -+ int ret; -+ -+ if (cnt >= sizeof(buf)) { -+ return -EINVAL; -+ } -+ -+ if (copy_from_user(&buf, ubuf, cnt)) { -+ return -EFAULT; -+ } -+ -+ buf[cnt] = 0; -+ -+ ret = kstrtol(buf, 10, &val); -+ if (ret < 0) { -+ return ret; -+ } -+ -+ if (val < 0) { -+ /* any negative input will disable counter */ -+ val = MALI_HW_CORE_NO_COUNTER; -+ } -+ -+ mali_l2_cache_core_set_counter_src(l2_core, src_id, (u32)val); -+ -+ *ppos += cnt; -+ return cnt; -+} -+ -+static ssize_t l2_all_counter_srcx_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) -+{ -+ char buf[64]; -+ long val; -+ int ret; -+ u32 l2_id; -+ struct mali_l2_cache_core *l2_cache; -+ -+ if (cnt >= sizeof(buf)) { -+ return -EINVAL; -+ } -+ -+ if (copy_from_user(&buf, ubuf, cnt)) { -+ return -EFAULT; -+ } -+ -+ buf[cnt] = 0; -+ -+ ret = kstrtol(buf, 10, &val); -+ if (ret < 0) { -+ return ret; -+ } -+ -+ if (val < 0) { -+ /* any negative input will disable counter */ -+ val = MALI_HW_CORE_NO_COUNTER; -+ } -+ -+ l2_id = 0; -+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); -+ while (NULL != l2_cache) { -+ mali_l2_cache_core_set_counter_src(l2_cache, src_id, (u32)val); -+ -+ /* try next L2 */ -+ l2_id++; -+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); -+ } -+ -+ *ppos += cnt; -+ return cnt; -+} -+ -+static ssize_t l2_l2x_counter_src0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 0); -+} -+ -+static ssize_t l2_l2x_counter_src1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_l2x_counter_srcx_read(filp, ubuf, cnt, ppos, 1); -+} -+ -+static ssize_t l2_l2x_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 0); -+} -+ -+static ssize_t l2_l2x_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_l2x_counter_srcx_write(filp, ubuf, cnt, ppos, 1); -+} -+ -+static ssize_t l2_all_counter_src0_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 0); -+} -+ -+static ssize_t l2_all_counter_src1_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_all_counter_srcx_write(filp, ubuf, cnt, ppos, 1); -+} -+ -+static const struct file_operations l2_l2x_counter_src0_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = l2_l2x_counter_src0_read, -+ .write = l2_l2x_counter_src0_write, -+}; -+ -+static const struct file_operations l2_l2x_counter_src1_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = l2_l2x_counter_src1_read, -+ .write = l2_l2x_counter_src1_write, -+}; -+ -+static const struct file_operations l2_all_counter_src0_fops = { -+ .owner = THIS_MODULE, -+ .write = l2_all_counter_src0_write, -+}; -+ -+static const struct file_operations l2_all_counter_src1_fops = { -+ .owner = THIS_MODULE, -+ .write = l2_all_counter_src1_write, -+}; -+ -+static ssize_t l2_l2x_counter_valx_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos, u32 src_id) -+{ -+ char buf[64]; -+ int r; -+ u32 src0 = 0; -+ u32 val0 = 0; -+ u32 src1 = 0; -+ u32 val1 = 0; -+ u32 val = -1; -+ struct mali_l2_cache_core *l2_core = (struct mali_l2_cache_core *)filp->private_data; -+ -+ mali_l2_cache_core_get_counter_values(l2_core, &src0, &val0, &src1, &val1); -+ -+ if (0 == src_id) { -+ if (MALI_HW_CORE_NO_COUNTER != val0) { -+ val = val0; -+ } -+ } else { -+ if (MALI_HW_CORE_NO_COUNTER != val1) { -+ val = val1; -+ } -+ } -+ -+ r = snprintf(buf, 64, "%u\n", val); -+ -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t l2_l2x_counter_val0_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 0); -+} -+ -+static ssize_t l2_l2x_counter_val1_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ return l2_l2x_counter_valx_read(filp, ubuf, cnt, ppos, 1); -+} -+ -+static const struct file_operations l2_l2x_counter_val0_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = l2_l2x_counter_val0_read, -+}; -+ -+static const struct file_operations l2_l2x_counter_val1_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = l2_l2x_counter_val1_read, -+}; -+ -+static ssize_t power_always_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ unsigned long val; -+ int ret; -+ char buf[32]; -+ -+ cnt = min(cnt, sizeof(buf) - 1); -+ if (copy_from_user(buf, ubuf, cnt)) { -+ return -EFAULT; -+ } -+ buf[cnt] = '\0'; -+ -+ ret = kstrtoul(buf, 10, &val); -+ if (0 != ret) { -+ return ret; -+ } -+ -+ /* Update setting (not exactly thread safe) */ -+ if (1 == val && MALI_FALSE == power_always_on_enabled) { -+ power_always_on_enabled = MALI_TRUE; -+ _mali_osk_pm_dev_ref_get_sync(); -+ } else if (0 == val && MALI_TRUE == power_always_on_enabled) { -+ power_always_on_enabled = MALI_FALSE; -+ _mali_osk_pm_dev_ref_put(); -+ } -+ -+ *ppos += cnt; -+ return cnt; -+} -+ -+static ssize_t power_always_on_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ if (MALI_TRUE == power_always_on_enabled) { -+ return simple_read_from_buffer(ubuf, cnt, ppos, "1\n", 2); -+ } else { -+ return simple_read_from_buffer(ubuf, cnt, ppos, "0\n", 2); -+ } -+} -+ -+static const struct file_operations power_always_on_fops = { -+ .owner = THIS_MODULE, -+ .read = power_always_on_read, -+ .write = power_always_on_write, -+}; -+ -+static ssize_t power_power_events_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_SUSPEND], strlen(mali_power_events[_MALI_DEVICE_SUSPEND]) - 1)) { -+ mali_pm_os_suspend(MALI_TRUE); -+ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_RESUME], strlen(mali_power_events[_MALI_DEVICE_RESUME]) - 1)) { -+ mali_pm_os_resume(); -+ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_PAUSE], strlen(mali_power_events[_MALI_DEVICE_DVFS_PAUSE]) - 1)) { -+ mali_dev_pause(); -+ } else if (!strncmp(ubuf, mali_power_events[_MALI_DEVICE_DVFS_RESUME], strlen(mali_power_events[_MALI_DEVICE_DVFS_RESUME]) - 1)) { -+ mali_dev_resume(); -+ } -+ *ppos += cnt; -+ return cnt; -+} -+ -+static loff_t power_power_events_seek(struct file *file, loff_t offset, int orig) -+{ -+ file->f_pos = offset; -+ return 0; -+} -+ -+static const struct file_operations power_power_events_fops = { -+ .owner = THIS_MODULE, -+ .write = power_power_events_write, -+ .llseek = power_power_events_seek, -+}; -+ -+#if MALI_STATE_TRACKING -+static int mali_seq_internal_state_show(struct seq_file *seq_file, void *v) -+{ -+ u32 len = 0; -+ u32 size; -+ char *buf; -+ -+ size = seq_get_buf(seq_file, &buf); -+ -+ if (!size) { -+ return -ENOMEM; -+ } -+ -+ /* Create the internal state dump. */ -+ len = snprintf(buf + len, size - len, "Mali device driver %s\n", SVN_REV_STRING); -+ len += snprintf(buf + len, size - len, "License: %s\n\n", MALI_KERNEL_LINUX_LICENSE); -+ -+ len += _mali_kernel_core_dump_state(buf + len, size - len); -+ -+ seq_commit(seq_file, len); -+ -+ return 0; -+} -+ -+static int mali_seq_internal_state_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, mali_seq_internal_state_show, NULL); -+} -+ -+static const struct file_operations mali_seq_internal_state_fops = { -+ .owner = THIS_MODULE, -+ .open = mali_seq_internal_state_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+#endif /* MALI_STATE_TRACKING */ -+ -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+static ssize_t profiling_record_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ int r; -+ -+ r = snprintf(buf, 64, "%u\n", _mali_internal_profiling_is_recording() ? 1 : 0); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t profiling_record_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ unsigned long val; -+ int ret; -+ -+ if (cnt >= sizeof(buf)) { -+ return -EINVAL; -+ } -+ -+ if (copy_from_user(&buf, ubuf, cnt)) { -+ return -EFAULT; -+ } -+ -+ buf[cnt] = 0; -+ -+ ret = kstrtoul(buf, 10, &val); -+ if (ret < 0) { -+ return ret; -+ } -+ -+ if (val != 0) { -+ u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* This can be made configurable at a later stage if we need to */ -+ -+ /* check if we are already recording */ -+ if (MALI_TRUE == _mali_internal_profiling_is_recording()) { -+ MALI_DEBUG_PRINT(3, ("Recording of profiling events already in progress\n")); -+ return -EFAULT; -+ } -+ -+ /* check if we need to clear out an old recording first */ -+ if (MALI_TRUE == _mali_internal_profiling_have_recording()) { -+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_clear()) { -+ MALI_DEBUG_PRINT(3, ("Failed to clear existing recording of profiling events\n")); -+ return -EFAULT; -+ } -+ } -+ -+ /* start recording profiling data */ -+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { -+ MALI_DEBUG_PRINT(3, ("Failed to start recording of profiling events\n")); -+ return -EFAULT; -+ } -+ -+ MALI_DEBUG_PRINT(3, ("Profiling recording started (max %u events)\n", limit)); -+ } else { -+ /* stop recording profiling data */ -+ u32 count = 0; -+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_stop(&count)) { -+ MALI_DEBUG_PRINT(2, ("Failed to stop recording of profiling events\n")); -+ return -EFAULT; -+ } -+ -+ MALI_DEBUG_PRINT(2, ("Profiling recording stopped (recorded %u events)\n", count)); -+ } -+ -+ *ppos += cnt; -+ return cnt; -+} -+ -+static const struct file_operations profiling_record_fops = { -+ .owner = THIS_MODULE, -+ .read = profiling_record_read, -+ .write = profiling_record_write, -+}; -+ -+static void *profiling_events_start(struct seq_file *s, loff_t *pos) -+{ -+ loff_t *spos; -+ -+ /* check if we have data avaiable */ -+ if (MALI_TRUE != _mali_internal_profiling_have_recording()) { -+ return NULL; -+ } -+ -+ spos = kmalloc(sizeof(loff_t), GFP_KERNEL); -+ if (NULL == spos) { -+ return NULL; -+ } -+ -+ *spos = *pos; -+ return spos; -+} -+ -+static void *profiling_events_next(struct seq_file *s, void *v, loff_t *pos) -+{ -+ loff_t *spos = v; -+ -+ /* check if we have data avaiable */ -+ if (MALI_TRUE != _mali_internal_profiling_have_recording()) { -+ return NULL; -+ } -+ -+ /* check if the next entry actually is avaiable */ -+ if (_mali_internal_profiling_get_count() <= (u32)(*spos + 1)) { -+ return NULL; -+ } -+ -+ *pos = ++*spos; -+ return spos; -+} -+ -+static void profiling_events_stop(struct seq_file *s, void *v) -+{ -+ kfree(v); -+} -+ -+static int profiling_events_show(struct seq_file *seq_file, void *v) -+{ -+ loff_t *spos = v; -+ u32 index; -+ u64 timestamp; -+ u32 event_id; -+ u32 data[5]; -+ -+ index = (u32) * spos; -+ -+ /* Retrieve all events */ -+ if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { -+ seq_printf(seq_file, "%llu %u %u %u %u %u %u\n", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); -+ return 0; -+ } -+ -+ return 0; -+} -+ -+static int profiling_events_show_human_readable(struct seq_file *seq_file, void *v) -+{ -+#define MALI_EVENT_ID_IS_HW(event_id) (((event_id & 0x00FF0000) >= MALI_PROFILING_EVENT_CHANNEL_GP0) && ((event_id & 0x00FF0000) <= MALI_PROFILING_EVENT_CHANNEL_PP7)) -+ -+ static u64 start_time = 0; -+ loff_t *spos = v; -+ u32 index; -+ u64 timestamp; -+ u32 event_id; -+ u32 data[5]; -+ -+ index = (u32) * spos; -+ -+ /* Retrieve all events */ -+ if (_MALI_OSK_ERR_OK == _mali_internal_profiling_get_event(index, ×tamp, &event_id, data)) { -+ seq_printf(seq_file, "%llu %u %u %u %u %u %u # ", timestamp, event_id, data[0], data[1], data[2], data[3], data[4]); -+ -+ if (0 == index) { -+ start_time = timestamp; -+ } -+ -+ seq_printf(seq_file, "[%06u] ", index); -+ -+ switch (event_id & 0x0F000000) { -+ case MALI_PROFILING_EVENT_TYPE_SINGLE: -+ seq_printf(seq_file, "SINGLE | "); -+ break; -+ case MALI_PROFILING_EVENT_TYPE_START: -+ seq_printf(seq_file, "START | "); -+ break; -+ case MALI_PROFILING_EVENT_TYPE_STOP: -+ seq_printf(seq_file, "STOP | "); -+ break; -+ case MALI_PROFILING_EVENT_TYPE_SUSPEND: -+ seq_printf(seq_file, "SUSPEND | "); -+ break; -+ case MALI_PROFILING_EVENT_TYPE_RESUME: -+ seq_printf(seq_file, "RESUME | "); -+ break; -+ default: -+ seq_printf(seq_file, "0x%01X | ", (event_id & 0x0F000000) >> 24); -+ break; -+ } -+ -+ switch (event_id & 0x00FF0000) { -+ case MALI_PROFILING_EVENT_CHANNEL_SOFTWARE: -+ seq_printf(seq_file, "SW | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_GP0: -+ seq_printf(seq_file, "GP0 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP0: -+ seq_printf(seq_file, "PP0 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP1: -+ seq_printf(seq_file, "PP1 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP2: -+ seq_printf(seq_file, "PP2 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP3: -+ seq_printf(seq_file, "PP3 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP4: -+ seq_printf(seq_file, "PP4 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP5: -+ seq_printf(seq_file, "PP5 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP6: -+ seq_printf(seq_file, "PP6 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_PP7: -+ seq_printf(seq_file, "PP7 | "); -+ break; -+ case MALI_PROFILING_EVENT_CHANNEL_GPU: -+ seq_printf(seq_file, "GPU | "); -+ break; -+ default: -+ seq_printf(seq_file, "0x%02X | ", (event_id & 0x00FF0000) >> 16); -+ break; -+ } -+ -+ if (MALI_EVENT_ID_IS_HW(event_id)) { -+ if (((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_START) || ((event_id & 0x0F000000) == MALI_PROFILING_EVENT_TYPE_STOP)) { -+ switch (event_id & 0x0000FFFF) { -+ case MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL: -+ seq_printf(seq_file, "PHYSICAL | "); -+ break; -+ case MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL: -+ seq_printf(seq_file, "VIRTUAL | "); -+ break; -+ default: -+ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); -+ break; -+ } -+ } else { -+ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); -+ } -+ } else { -+ seq_printf(seq_file, "0x%04X | ", event_id & 0x0000FFFF); -+ } -+ -+ seq_printf(seq_file, "T0 + 0x%016llX\n", timestamp - start_time); -+ -+ return 0; -+ } -+ -+ return 0; -+} -+ -+static const struct seq_operations profiling_events_seq_ops = { -+ .start = profiling_events_start, -+ .next = profiling_events_next, -+ .stop = profiling_events_stop, -+ .show = profiling_events_show -+}; -+ -+static int profiling_events_open(struct inode *inode, struct file *file) -+{ -+ return seq_open(file, &profiling_events_seq_ops); -+} -+ -+static const struct file_operations profiling_events_fops = { -+ .owner = THIS_MODULE, -+ .open = profiling_events_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+static const struct seq_operations profiling_events_human_readable_seq_ops = { -+ .start = profiling_events_start, -+ .next = profiling_events_next, -+ .stop = profiling_events_stop, -+ .show = profiling_events_show_human_readable -+}; -+ -+static int profiling_events_human_readable_open(struct inode *inode, struct file *file) -+{ -+ return seq_open(file, &profiling_events_human_readable_seq_ops); -+} -+ -+static const struct file_operations profiling_events_human_readable_fops = { -+ .owner = THIS_MODULE, -+ .open = profiling_events_human_readable_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+#endif -+ -+static int memory_debugfs_show(struct seq_file *s, void *private_data) -+{ -+#ifdef MALI_MEM_SWAP_TRACKING -+ seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s %-10s \n"\ -+ "=================================================================================================================================\n", -+ "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", -+ "external_mem", "ump_mem", "dma_mem", "swap_mem"); -+#else -+ seq_printf(s, " %-25s %-10s %-10s %-15s %-15s %-10s %-10s \n"\ -+ "========================================================================================================================\n", -+ "Name (:bytes)", "pid", "mali_mem", "max_mali_mem", -+ "external_mem", "ump_mem", "dma_mem"); -+#endif -+ mali_session_memory_tracking(s); -+ return 0; -+} -+ -+static int memory_debugfs_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, memory_debugfs_show, inode->i_private); -+} -+ -+static const struct file_operations memory_usage_fops = { -+ .owner = THIS_MODULE, -+ .open = memory_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+static ssize_t utilization_gp_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ size_t r; -+ u32 uval = _mali_ukk_utilization_gp_pp(); -+ -+ r = snprintf(buf, 64, "%u\n", uval); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t utilization_gp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ size_t r; -+ u32 uval = _mali_ukk_utilization_gp(); -+ -+ r = snprintf(buf, 64, "%u\n", uval); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t utilization_pp_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ size_t r; -+ u32 uval = _mali_ukk_utilization_pp(); -+ -+ r = snprintf(buf, 64, "%u\n", uval); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+ -+static const struct file_operations utilization_gp_pp_fops = { -+ .owner = THIS_MODULE, -+ .read = utilization_gp_pp_read, -+}; -+ -+static const struct file_operations utilization_gp_fops = { -+ .owner = THIS_MODULE, -+ .read = utilization_gp_read, -+}; -+ -+static const struct file_operations utilization_pp_fops = { -+ .owner = THIS_MODULE, -+ .read = utilization_pp_read, -+}; -+ -+static ssize_t user_settings_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ unsigned long val; -+ int ret; -+ _mali_uk_user_setting_t setting; -+ char buf[32]; -+ -+ cnt = min(cnt, sizeof(buf) - 1); -+ if (copy_from_user(buf, ubuf, cnt)) { -+ return -EFAULT; -+ } -+ buf[cnt] = '\0'; -+ -+ ret = kstrtoul(buf, 10, &val); -+ if (0 != ret) { -+ return ret; -+ } -+ -+ /* Update setting */ -+ setting = (_mali_uk_user_setting_t)(filp->private_data); -+ mali_set_user_setting(setting, val); -+ -+ *ppos += cnt; -+ return cnt; -+} -+ -+static ssize_t user_settings_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ size_t r; -+ u32 value; -+ _mali_uk_user_setting_t setting; -+ -+ setting = (_mali_uk_user_setting_t)(filp->private_data); -+ value = mali_get_user_setting(setting); -+ -+ r = snprintf(buf, 64, "%u\n", value); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static const struct file_operations user_settings_fops = { -+ .owner = THIS_MODULE, -+ .open = open_copy_private_data, -+ .read = user_settings_read, -+ .write = user_settings_write, -+}; -+ -+static int mali_sysfs_user_settings_register(void) -+{ -+ struct dentry *mali_user_settings_dir = debugfs_create_dir("userspace_settings", mali_debugfs_dir); -+ -+ if (mali_user_settings_dir != NULL) { -+ long i; -+ for (i = 0; i < _MALI_UK_USER_SETTING_MAX; i++) { -+ debugfs_create_file(_mali_uk_user_setting_descriptions[i], -+ 0600, mali_user_settings_dir, (void *)i, -+ &user_settings_fops); -+ } -+ } -+ -+ return 0; -+} -+ -+static ssize_t pp_num_cores_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) -+{ -+ int ret; -+ char buffer[32]; -+ unsigned long val; -+ -+ if (count >= sizeof(buffer)) { -+ return -ENOMEM; -+ } -+ -+ if (copy_from_user(&buffer[0], buf, count)) { -+ return -EFAULT; -+ } -+ buffer[count] = '\0'; -+ -+ ret = kstrtoul(&buffer[0], 10, &val); -+ if (0 != ret) { -+ return -EINVAL; -+ } -+ -+ ret = mali_executor_set_perf_level(val, MALI_TRUE); /* override even if core scaling is disabled */ -+ if (ret) { -+ return ret; -+ } -+ -+ *offp += count; -+ return count; -+} -+ -+static ssize_t pp_num_cores_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) -+{ -+ int r; -+ char buffer[64]; -+ -+ r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_enabled()); -+ -+ return simple_read_from_buffer(buf, count, offp, buffer, r); -+} -+ -+static const struct file_operations pp_num_cores_enabled_fops = { -+ .owner = THIS_MODULE, -+ .write = pp_num_cores_enabled_write, -+ .read = pp_num_cores_enabled_read, -+ .llseek = default_llseek, -+}; -+ -+static ssize_t pp_num_cores_total_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) -+{ -+ int r; -+ char buffer[64]; -+ -+ r = snprintf(buffer, 64, "%u\n", mali_executor_get_num_cores_total()); -+ -+ return simple_read_from_buffer(buf, count, offp, buffer, r); -+} -+ -+static const struct file_operations pp_num_cores_total_fops = { -+ .owner = THIS_MODULE, -+ .read = pp_num_cores_total_read, -+}; -+ -+static ssize_t pp_core_scaling_enabled_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) -+{ -+ int ret; -+ char buffer[32]; -+ unsigned long val; -+ -+ if (count >= sizeof(buffer)) { -+ return -ENOMEM; -+ } -+ -+ if (copy_from_user(&buffer[0], buf, count)) { -+ return -EFAULT; -+ } -+ buffer[count] = '\0'; -+ -+ ret = kstrtoul(&buffer[0], 10, &val); -+ if (0 != ret) { -+ return -EINVAL; -+ } -+ -+ switch (val) { -+ case 1: -+ mali_executor_core_scaling_enable(); -+ break; -+ case 0: -+ mali_executor_core_scaling_disable(); -+ break; -+ default: -+ return -EINVAL; -+ break; -+ } -+ -+ *offp += count; -+ return count; -+} -+ -+static ssize_t pp_core_scaling_enabled_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) -+{ -+ return simple_read_from_buffer(buf, count, offp, mali_executor_core_scaling_is_enabled() ? "1\n" : "0\n", 2); -+} -+static const struct file_operations pp_core_scaling_enabled_fops = { -+ .owner = THIS_MODULE, -+ .write = pp_core_scaling_enabled_write, -+ .read = pp_core_scaling_enabled_read, -+ .llseek = default_llseek, -+}; -+ -+static ssize_t version_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) -+{ -+ int r = 0; -+ char buffer[64]; -+ -+ switch (mali_kernel_core_get_product_id()) { -+ case _MALI_PRODUCT_ID_MALI200: -+ r = snprintf(buffer, 64, "Mali-200\n"); -+ break; -+ case _MALI_PRODUCT_ID_MALI300: -+ r = snprintf(buffer, 64, "Mali-300\n"); -+ break; -+ case _MALI_PRODUCT_ID_MALI400: -+ r = snprintf(buffer, 64, "Mali-400 MP\n"); -+ break; -+ case _MALI_PRODUCT_ID_MALI450: -+ r = snprintf(buffer, 64, "Mali-450 MP\n"); -+ break; -+ case _MALI_PRODUCT_ID_MALI470: -+ r = snprintf(buffer, 64, "Mali-470 MP\n"); -+ break; -+ case _MALI_PRODUCT_ID_UNKNOWN: -+ return -EINVAL; -+ break; -+ }; -+ -+ return simple_read_from_buffer(buf, count, offp, buffer, r); -+} -+ -+static const struct file_operations version_fops = { -+ .owner = THIS_MODULE, -+ .read = version_read, -+}; -+ -+#if defined(DEBUG) -+static int timeline_debugfs_show(struct seq_file *s, void *private_data) -+{ -+ struct mali_session_data *session, *tmp; -+ u32 session_seq = 1; -+ -+ seq_printf(s, "timeline system info: \n=================\n\n"); -+ -+ mali_session_lock(); -+ MALI_SESSION_FOREACH(session, tmp, link) { -+ seq_printf(s, "session %d <%p> start:\n", session_seq, session); -+ mali_timeline_debug_print_system(session->timeline_system, s); -+ seq_printf(s, "session %d end\n\n\n", session_seq++); -+ } -+ mali_session_unlock(); -+ -+ return 0; -+} -+ -+static int timeline_debugfs_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, timeline_debugfs_show, inode->i_private); -+} -+ -+static const struct file_operations timeline_dump_fops = { -+ .owner = THIS_MODULE, -+ .open = timeline_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release -+}; -+#endif -+ -+int mali_sysfs_register(const char *mali_dev_name) -+{ -+ mali_debugfs_dir = debugfs_create_dir(mali_dev_name, NULL); -+ if (ERR_PTR(-ENODEV) == mali_debugfs_dir) { -+ /* Debugfs not supported. */ -+ mali_debugfs_dir = NULL; -+ } else { -+ if (NULL != mali_debugfs_dir) { -+ /* Debugfs directory created successfully; create files now */ -+ struct dentry *mali_power_dir; -+ struct dentry *mali_gp_dir; -+ struct dentry *mali_pp_dir; -+ struct dentry *mali_l2_dir; -+ struct dentry *mali_profiling_dir; -+ -+ debugfs_create_file("version", 0400, mali_debugfs_dir, NULL, &version_fops); -+ -+ mali_power_dir = debugfs_create_dir("power", mali_debugfs_dir); -+ if (mali_power_dir != NULL) { -+ debugfs_create_file("always_on", 0600, mali_power_dir, NULL, &power_always_on_fops); -+ debugfs_create_file("power_events", 0200, mali_power_dir, NULL, &power_power_events_fops); -+ } -+ -+ mali_gp_dir = debugfs_create_dir("gp", mali_debugfs_dir); -+ if (mali_gp_dir != NULL) { -+ u32 num_groups; -+ long i; -+ -+ num_groups = mali_group_get_glob_num_groups(); -+ for (i = 0; i < num_groups; i++) { -+ struct mali_group *group = mali_group_get_glob_group(i); -+ -+ struct mali_gp_core *gp_core = mali_group_get_gp_core(group); -+ if (NULL != gp_core) { -+ struct dentry *mali_gp_gpx_dir; -+ mali_gp_gpx_dir = debugfs_create_dir("gp0", mali_gp_dir); -+ if (NULL != mali_gp_gpx_dir) { -+ debugfs_create_file("base_addr", 0400, mali_gp_gpx_dir, &gp_core->hw_core, &hw_core_base_addr_fops); -+ debugfs_create_file("enabled", 0600, mali_gp_gpx_dir, group, &group_enabled_fops); -+ } -+ break; /* no need to look for any other GP cores */ -+ } -+ -+ } -+ } -+ -+ mali_pp_dir = debugfs_create_dir("pp", mali_debugfs_dir); -+ if (mali_pp_dir != NULL) { -+ u32 num_groups; -+ long i; -+ -+ debugfs_create_file("num_cores_total", 0400, mali_pp_dir, NULL, &pp_num_cores_total_fops); -+ debugfs_create_file("num_cores_enabled", 0600, mali_pp_dir, NULL, &pp_num_cores_enabled_fops); -+ debugfs_create_file("core_scaling_enabled", 0600, mali_pp_dir, NULL, &pp_core_scaling_enabled_fops); -+ -+ num_groups = mali_group_get_glob_num_groups(); -+ for (i = 0; i < num_groups; i++) { -+ struct mali_group *group = mali_group_get_glob_group(i); -+ -+ struct mali_pp_core *pp_core = mali_group_get_pp_core(group); -+ if (NULL != pp_core) { -+ char buf[16]; -+ struct dentry *mali_pp_ppx_dir; -+ _mali_osk_snprintf(buf, sizeof(buf), "pp%u", mali_pp_core_get_id(pp_core)); -+ mali_pp_ppx_dir = debugfs_create_dir(buf, mali_pp_dir); -+ if (NULL != mali_pp_ppx_dir) { -+ debugfs_create_file("base_addr", 0400, mali_pp_ppx_dir, &pp_core->hw_core, &hw_core_base_addr_fops); -+ if (!mali_group_is_virtual(group)) { -+ debugfs_create_file("enabled", 0600, mali_pp_ppx_dir, group, &group_enabled_fops); -+ } -+ } -+ } -+ } -+ } -+ -+ mali_l2_dir = debugfs_create_dir("l2", mali_debugfs_dir); -+ if (mali_l2_dir != NULL) { -+ struct dentry *mali_l2_all_dir; -+ u32 l2_id; -+ struct mali_l2_cache_core *l2_cache; -+ -+ mali_l2_all_dir = debugfs_create_dir("all", mali_l2_dir); -+ if (mali_l2_all_dir != NULL) { -+ debugfs_create_file("counter_src0", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src0_fops); -+ debugfs_create_file("counter_src1", 0200, mali_l2_all_dir, NULL, &l2_all_counter_src1_fops); -+ } -+ -+ l2_id = 0; -+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); -+ while (NULL != l2_cache) { -+ char buf[16]; -+ struct dentry *mali_l2_l2x_dir; -+ _mali_osk_snprintf(buf, sizeof(buf), "l2%u", l2_id); -+ mali_l2_l2x_dir = debugfs_create_dir(buf, mali_l2_dir); -+ if (NULL != mali_l2_l2x_dir) { -+ debugfs_create_file("counter_src0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src0_fops); -+ debugfs_create_file("counter_src1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_src1_fops); -+ debugfs_create_file("counter_val0", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val0_fops); -+ debugfs_create_file("counter_val1", 0600, mali_l2_l2x_dir, l2_cache, &l2_l2x_counter_val1_fops); -+ debugfs_create_file("base_addr", 0400, mali_l2_l2x_dir, &l2_cache->hw_core, &hw_core_base_addr_fops); -+ } -+ -+ /* try next L2 */ -+ l2_id++; -+ l2_cache = mali_l2_cache_core_get_glob_l2_core(l2_id); -+ } -+ } -+ -+ debugfs_create_file("gpu_memory", 0444, mali_debugfs_dir, NULL, &memory_usage_fops); -+ -+ debugfs_create_file("utilization_gp_pp", 0400, mali_debugfs_dir, NULL, &utilization_gp_pp_fops); -+ debugfs_create_file("utilization_gp", 0400, mali_debugfs_dir, NULL, &utilization_gp_fops); -+ debugfs_create_file("utilization_pp", 0400, mali_debugfs_dir, NULL, &utilization_pp_fops); -+ -+ mali_profiling_dir = debugfs_create_dir("profiling", mali_debugfs_dir); -+ if (mali_profiling_dir != NULL) { -+ u32 max_sub_jobs; -+ long i; -+ struct dentry *mali_profiling_gp_dir; -+ struct dentry *mali_profiling_pp_dir; -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+ struct dentry *mali_profiling_proc_dir; -+#endif -+ /* -+ * Create directory where we can set GP HW counters. -+ */ -+ mali_profiling_gp_dir = debugfs_create_dir("gp", mali_profiling_dir); -+ if (mali_profiling_gp_dir != NULL) { -+ debugfs_create_file("counter_src0", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(0), &profiling_counter_src_fops); -+ debugfs_create_file("counter_src1", 0600, mali_profiling_gp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_GP(1), &profiling_counter_src_fops); -+ } -+ -+ /* -+ * Create directory where we can set PP HW counters. -+ * Possible override with specific HW counters for a particular sub job -+ * (Disable core scaling before using the override!) -+ */ -+ mali_profiling_pp_dir = debugfs_create_dir("pp", mali_profiling_dir); -+ if (mali_profiling_pp_dir != NULL) { -+ debugfs_create_file("counter_src0", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(0), &profiling_counter_src_fops); -+ debugfs_create_file("counter_src1", 0600, mali_profiling_pp_dir, (void *)PRIVATE_DATA_COUNTER_MAKE_PP(1), &profiling_counter_src_fops); -+ } -+ -+ max_sub_jobs = mali_executor_get_num_cores_total(); -+ for (i = 0; i < max_sub_jobs; i++) { -+ char buf[16]; -+ struct dentry *mali_profiling_pp_x_dir; -+ _mali_osk_snprintf(buf, sizeof(buf), "%u", i); -+ mali_profiling_pp_x_dir = debugfs_create_dir(buf, mali_profiling_pp_dir); -+ if (NULL != mali_profiling_pp_x_dir) { -+ debugfs_create_file("counter_src0", -+ 0600, mali_profiling_pp_x_dir, -+ (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(0, i), -+ &profiling_counter_src_fops); -+ debugfs_create_file("counter_src1", -+ 0600, mali_profiling_pp_x_dir, -+ (void *)PRIVATE_DATA_COUNTER_MAKE_PP_SUB_JOB(1, i), -+ &profiling_counter_src_fops); -+ } -+ } -+ -+#if defined(CONFIG_MALI400_INTERNAL_PROFILING) -+ mali_profiling_proc_dir = debugfs_create_dir("proc", mali_profiling_dir); -+ if (mali_profiling_proc_dir != NULL) { -+ struct dentry *mali_profiling_proc_default_dir = debugfs_create_dir("default", mali_profiling_proc_dir); -+ if (mali_profiling_proc_default_dir != NULL) { -+ debugfs_create_file("enable", 0600, mali_profiling_proc_default_dir, (void *)_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, &user_settings_fops); -+ } -+ } -+ debugfs_create_file("record", 0600, mali_profiling_dir, NULL, &profiling_record_fops); -+ debugfs_create_file("events", 0400, mali_profiling_dir, NULL, &profiling_events_fops); -+ debugfs_create_file("events_human_readable", 0400, mali_profiling_dir, NULL, &profiling_events_human_readable_fops); -+#endif -+ } -+ -+#if MALI_STATE_TRACKING -+ debugfs_create_file("state_dump", 0400, mali_debugfs_dir, NULL, &mali_seq_internal_state_fops); -+#endif -+ -+#if defined(DEBUG) -+ debugfs_create_file("timeline_dump", 0400, mali_debugfs_dir, NULL, &timeline_dump_fops); -+#endif -+ if (mali_sysfs_user_settings_register()) { -+ /* Failed to create the debugfs entries for the user settings DB. */ -+ MALI_DEBUG_PRINT(2, ("Failed to create user setting debugfs files. Ignoring...\n")); -+ } -+ } -+ } -+ -+ /* Success! */ -+ return 0; -+} -+ -+int mali_sysfs_unregister(void) -+{ -+ if (NULL != mali_debugfs_dir) { -+ debugfs_remove_recursive(mali_debugfs_dir); -+ } -+ return 0; -+} -+ -+#else /* MALI_LICENSE_IS_GPL */ -+ -+/* Dummy implementations for non-GPL */ -+ -+int mali_sysfs_register(struct mali_dev *device, dev_t dev, const char *mali_dev_name) -+{ -+ return 0; -+} -+ -+int mali_sysfs_unregister(void) -+{ -+ return 0; -+} -+ -+#endif /* MALI_LICENSE_IS_GPL */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h -new file mode 100755 -index 000000000..91580a87c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_kernel_sysfs.h -@@ -0,0 +1,29 @@ -+/* -+ * Copyright (C) 2011-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_KERNEL_SYSFS_H__ -+#define __MALI_KERNEL_SYSFS_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include -+ -+#define MALI_PROC_DIR "driver/mali" -+ -+int mali_sysfs_register(const char *mali_dev_name); -+int mali_sysfs_unregister(void); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_KERNEL_LINUX_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h -new file mode 100755 -index 000000000..222260823 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_linux_trace.h -@@ -0,0 +1,161 @@ -+/* -+ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#if !defined (MALI_LINUX_TRACE_H) || defined (TRACE_HEADER_MULTI_READ) -+#define MALI_LINUX_TRACE_H -+ -+#include -+ -+#include -+#include -+ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM mali -+ -+#define TRACE_INCLUDE_PATH . -+#define TRACE_INCLUDE_FILE mali_linux_trace -+ -+/** -+ * Define the tracepoint used to communicate the status of a GPU. Called -+ * when a GPU turns on or turns off. -+ * -+ * @param event_id The type of the event. This parameter is a bitfield -+ * encoding the type of the event. -+ * -+ * @param d0 First data parameter. -+ * @param d1 Second data parameter. -+ * @param d2 Third data parameter. -+ * @param d3 Fourth data parameter. -+ * @param d4 Fifth data parameter. -+ */ -+TRACE_EVENT(mali_timeline_event, -+ -+ TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, -+ unsigned int d2, unsigned int d3, unsigned int d4), -+ -+ TP_ARGS(event_id, d0, d1, d2, d3, d4), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, event_id) -+ __field(unsigned int, d0) -+ __field(unsigned int, d1) -+ __field(unsigned int, d2) -+ __field(unsigned int, d3) -+ __field(unsigned int, d4) -+ ), -+ -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ __entry->d0 = d0; -+ __entry->d1 = d1; -+ __entry->d2 = d2; -+ __entry->d3 = d3; -+ __entry->d4 = d4; -+ ), -+ -+ TP_printk("event=%d", __entry->event_id) -+ ); -+ -+/** -+ * Define a tracepoint used to regsiter the value of a hardware counter. -+ * Hardware counters belonging to the vertex or fragment processor are -+ * reported via this tracepoint each frame, whilst L2 cache hardware -+ * counters are reported continuously. -+ * -+ * @param counter_id The counter ID. -+ * @param value The value of the counter. -+ */ -+TRACE_EVENT(mali_hw_counter, -+ -+ TP_PROTO(unsigned int counter_id, unsigned int value), -+ -+ TP_ARGS(counter_id, value), -+ -+ TP_STRUCT__entry( -+ __field(unsigned int, counter_id) -+ __field(unsigned int, value) -+ ), -+ -+ TP_fast_assign( -+ __entry->counter_id = counter_id; -+ ), -+ -+ TP_printk("event %d = %d", __entry->counter_id, __entry->value) -+ ); -+ -+/** -+ * Define a tracepoint used to send a bundle of software counters. -+ * -+ * @param counters The bundle of counters. -+ */ -+TRACE_EVENT(mali_sw_counters, -+ -+ TP_PROTO(pid_t pid, pid_t tid, void *surface_id, unsigned int *counters), -+ -+ TP_ARGS(pid, tid, surface_id, counters), -+ -+ TP_STRUCT__entry( -+ __field(pid_t, pid) -+ __field(pid_t, tid) -+ __field(void *, surface_id) -+ __field(unsigned int *, counters) -+ ), -+ -+ TP_fast_assign( -+ __entry->pid = pid; -+ __entry->tid = tid; -+ __entry->surface_id = surface_id; -+ __entry->counters = counters; -+ ), -+ -+ TP_printk("counters were %s", __entry->counters == NULL ? "NULL" : "not NULL") -+ ); -+ -+/** -+ * Define a tracepoint used to gather core activity for systrace -+ * @param pid The process id for which the core activity originates from -+ * @param active If the core is active (1) or not (0) -+ * @param core_type The type of core active, either GP (1) or PP (0) -+ * @param core_id The core id that is active for the core_type -+ * @param frame_builder_id The frame builder id associated with this core activity -+ * @param flush_id The flush id associated with this core activity -+ */ -+TRACE_EVENT(mali_core_active, -+ -+ TP_PROTO(pid_t pid, unsigned int active, unsigned int core_type, unsigned int core_id, unsigned int frame_builder_id, unsigned int flush_id), -+ -+ TP_ARGS(pid, active, core_type, core_id, frame_builder_id, flush_id), -+ -+ TP_STRUCT__entry( -+ __field(pid_t, pid) -+ __field(unsigned int, active) -+ __field(unsigned int, core_type) -+ __field(unsigned int, core_id) -+ __field(unsigned int, frame_builder_id) -+ __field(unsigned int, flush_id) -+ ), -+ -+ TP_fast_assign( -+ __entry->pid = pid; -+ __entry->active = active; -+ __entry->core_type = core_type; -+ __entry->core_id = core_id; -+ __entry->frame_builder_id = frame_builder_id; -+ __entry->flush_id = flush_id; -+ ), -+ -+ TP_printk("%s|%d|%s%i:%x|%d", __entry->active ? "S" : "F", __entry->pid, __entry->core_type ? "GP" : "PP", __entry->core_id, __entry->flush_id, __entry->frame_builder_id) -+ ); -+ -+#endif /* MALI_LINUX_TRACE_H */ -+ -+/* This part must exist outside the header guard. */ -+#include -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c -new file mode 100755 -index 000000000..dfc769e6c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.c -@@ -0,0 +1,531 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_executor.h" -+ -+#include "mali_memory.h" -+#include "mali_memory_os_alloc.h" -+#include "mali_memory_block_alloc.h" -+#include "mali_memory_util.h" -+#include "mali_memory_virtual.h" -+#include "mali_memory_manager.h" -+#include "mali_memory_cow.h" -+#include "mali_memory_swap_alloc.h" -+#include "mali_memory_defer_bind.h" -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+#include "mali_memory_secure.h" -+#endif -+ -+extern unsigned int mali_dedicated_mem_size; -+extern unsigned int mali_shared_mem_size; -+ -+#define MALI_VM_NUM_FAULT_PREFETCH (0x8) -+ -+static void mali_mem_vma_open(struct vm_area_struct *vma) -+{ -+ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; -+ MALI_DEBUG_PRINT(4, ("Open called on vma %p\n", vma)); -+ -+ /* If need to share the allocation, add ref_count here */ -+ mali_allocation_ref(alloc); -+ return; -+} -+static void mali_mem_vma_close(struct vm_area_struct *vma) -+{ -+ /* If need to share the allocation, unref ref_count here */ -+ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; -+ -+ mali_allocation_unref(&alloc); -+ vma->vm_private_data = NULL; -+} -+ -+static vm_fault_t mali_mem_vma_fault(struct vm_fault *vmf) -+{ -+ struct vm_area_struct *vma = vmf->vma; -+ mali_mem_allocation *alloc = (mali_mem_allocation *)vma->vm_private_data; -+ mali_mem_backend *mem_bkend = NULL; -+ int ret; -+ int prefetch_num = MALI_VM_NUM_FAULT_PREFETCH; -+ -+ unsigned long address = (unsigned long)vmf->address; -+ MALI_DEBUG_ASSERT(alloc->backend_handle); -+ MALI_DEBUG_ASSERT((unsigned long)alloc->cpu_mapping.addr <= address); -+ -+ /* Get backend memory & Map on CPU */ -+ mutex_lock(&mali_idr_mutex); -+ if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { -+ MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); -+ mutex_unlock(&mali_idr_mutex); -+ return VM_FAULT_SIGBUS; -+ } -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(mem_bkend->type == alloc->type); -+ -+ if ((mem_bkend->type == MALI_MEM_COW && (MALI_MEM_BACKEND_FLAG_SWAP_COWED != -+ (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) && -+ (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE)) { -+ /*check if use page fault to do COW*/ -+ MALI_DEBUG_PRINT(4, ("mali_vma_fault: do cow allocate on demand!, address=0x%x\n", address)); -+ mutex_lock(&mem_bkend->mutex); -+ ret = mali_mem_cow_allocate_on_demand(mem_bkend, -+ (address - vma->vm_start) / PAGE_SIZE); -+ mutex_unlock(&mem_bkend->mutex); -+ -+ if (ret != _MALI_OSK_ERR_OK) { -+ return VM_FAULT_OOM; -+ } -+ prefetch_num = 1; -+ -+ /* handle COW modified range cpu mapping -+ we zap the mapping in cow_modify_range, it will trigger page fault -+ when CPU access it, so here we map it to CPU*/ -+ mutex_lock(&mem_bkend->mutex); -+ ret = mali_mem_cow_cpu_map_pages_locked(mem_bkend, vma, address, prefetch_num); -+ mutex_unlock(&mem_bkend->mutex); -+ -+ if (unlikely(ret != _MALI_OSK_ERR_OK)) { -+ return VM_FAULT_SIGBUS; -+ } -+ } else if ((mem_bkend->type == MALI_MEM_SWAP) || -+ (mem_bkend->type == MALI_MEM_COW && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { -+ u32 offset_in_bkend = (address - vma->vm_start) / PAGE_SIZE; -+ int ret = _MALI_OSK_ERR_OK; -+ -+ mutex_lock(&mem_bkend->mutex); -+ if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE) { -+ ret = mali_mem_swap_cow_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); -+ } else { -+ ret = mali_mem_swap_allocate_page_on_demand(mem_bkend, offset_in_bkend, &vmf->page); -+ } -+ mutex_unlock(&mem_bkend->mutex); -+ -+ if (ret != _MALI_OSK_ERR_OK) { -+ MALI_DEBUG_PRINT(2, ("Mali swap memory page fault process failed, address=0x%x\n", address)); -+ return VM_FAULT_OOM; -+ } else { -+ return VM_FAULT_LOCKED; -+ } -+ } else { -+ MALI_PRINT_ERROR(("Mali vma fault! It never happen, indicating some logic errors in caller.\n")); -+ /*NOT support yet or OOM*/ -+ return VM_FAULT_OOM; -+ } -+ return VM_FAULT_NOPAGE; -+} -+ -+static struct vm_operations_struct mali_kernel_vm_ops = { -+ .open = mali_mem_vma_open, -+ .close = mali_mem_vma_close, -+ .fault = mali_mem_vma_fault, -+}; -+ -+ -+/** @ map mali allocation to CPU address -+* -+* Supported backend types: -+* --MALI_MEM_OS -+* -- need to add COW? -+ *Not supported backend types: -+* -_MALI_MEMORY_BIND_BACKEND_UMP -+* -_MALI_MEMORY_BIND_BACKEND_DMA_BUF -+* -_MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY -+* -+*/ -+int mali_mmap(struct file *filp, struct vm_area_struct *vma) -+{ -+ struct mali_session_data *session; -+ mali_mem_allocation *mali_alloc = NULL; -+ u32 mali_addr = vma->vm_pgoff << PAGE_SHIFT; -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_backend *mem_bkend = NULL; -+ int ret = -EFAULT; -+ -+ session = (struct mali_session_data *)filp->private_data; -+ if (NULL == session) { -+ MALI_PRINT_ERROR(("mmap called without any session data available\n")); -+ return -EFAULT; -+ } -+ -+ MALI_DEBUG_PRINT(4, ("MMap() handler: start=0x%08X, phys=0x%08X, size=0x%08X vma->flags 0x%08x\n", -+ (unsigned int)vma->vm_start, (unsigned int)(vma->vm_pgoff << PAGE_SHIFT), -+ (unsigned int)(vma->vm_end - vma->vm_start), vma->vm_flags)); -+ -+ /* Operations used on any memory system */ -+ /* do not need to anything in vm open/close now */ -+ -+ /* find mali allocation structure by vaddress*/ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); -+ if (likely(mali_vma_node)) { -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); -+ if (unlikely(mali_addr != mali_vma_node->vm_node.start)) { -+ /* only allow to use start address for mmap */ -+ MALI_DEBUG_PRINT(1, ("mali_addr != mali_vma_node->vm_node.start\n")); -+ return -EFAULT; -+ } -+ } else { -+ MALI_DEBUG_ASSERT(NULL == mali_vma_node); -+ return -EFAULT; -+ } -+ -+ mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; -+ -+ if (mali_alloc->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { -+ MALI_DEBUG_PRINT(1, ("ERROR : trying to access varying memory by CPU!\n")); -+ return -EFAULT; -+ } -+ -+ /* Get backend memory & Map on CPU */ -+ mutex_lock(&mali_idr_mutex); -+ if (!(mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle))) { -+ MALI_DEBUG_PRINT(1, ("Can't find memory backend in mmap!\n")); -+ mutex_unlock(&mali_idr_mutex); -+ return -EFAULT; -+ } -+ mutex_unlock(&mali_idr_mutex); -+ -+ if (!(MALI_MEM_SWAP == mali_alloc->type || -+ (MALI_MEM_COW == mali_alloc->type && (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { -+ /* Set some bits which indicate that, the memory is IO memory, meaning -+ * that no paging is to be performed and the memory should not be -+ * included in crash dumps. And that the memory is reserved, meaning -+ * that it's present and can never be paged out (see also previous -+ * entry) -+ */ -+ vma->vm_flags |= VM_IO; -+ vma->vm_flags |= VM_DONTCOPY; -+ vma->vm_flags |= VM_PFNMAP; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) -+ vma->vm_flags |= VM_RESERVED; -+#else -+ vma->vm_flags |= VM_DONTDUMP; -+ vma->vm_flags |= VM_DONTEXPAND; -+#endif -+ } else if (MALI_MEM_SWAP == mali_alloc->type) { -+ vma->vm_pgoff = mem_bkend->start_idx; -+ } -+ -+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); -+ vma->vm_ops = &mali_kernel_vm_ops; -+ -+ mali_alloc->cpu_mapping.addr = (void __user *)vma->vm_start; -+ -+ /* If it's a copy-on-write mapping, map to read only */ -+ if (!(vma->vm_flags & VM_WRITE)) { -+ MALI_DEBUG_PRINT(4, ("mmap allocation with read only !\n")); -+ /* add VM_WRITE for do_page_fault will check this when a write fault */ -+ vma->vm_flags |= VM_WRITE | VM_READ; -+ vma->vm_page_prot = PAGE_READONLY; -+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); -+ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE; -+ goto out; -+ } -+ -+ if (mem_bkend->type == MALI_MEM_OS) { -+ ret = mali_mem_os_cpu_map(mem_bkend, vma); -+ } else if (mem_bkend->type == MALI_MEM_COW && -+ (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { -+ ret = mali_mem_cow_cpu_map(mem_bkend, vma); -+ } else if (mem_bkend->type == MALI_MEM_BLOCK) { -+ ret = mali_mem_block_cpu_map(mem_bkend, vma); -+ } else if ((mem_bkend->type == MALI_MEM_SWAP) || (mem_bkend->type == MALI_MEM_COW && -+ (MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)))) { -+ /*For swappable memory, CPU page table will be created by page fault handler. */ -+ ret = 0; -+ } else if (mem_bkend->type == MALI_MEM_SECURE) { -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ ret = mali_mem_secure_cpu_map(mem_bkend, vma); -+#else -+ MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory\n")); -+ return -EFAULT; -+#endif -+ } else { -+ /* Not support yet*/ -+ MALI_DEBUG_PRINT_ERROR(("Invalid type of backend memory! \n")); -+ return -EFAULT; -+ } -+ -+ if (ret != 0) { -+ MALI_DEBUG_PRINT(1, ("ret != 0\n")); -+ return -EFAULT; -+ } -+out: -+ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == mali_alloc->magic); -+ -+ vma->vm_private_data = (void *)mali_alloc; -+ mali_alloc->cpu_mapping.vma = vma; -+ -+ mali_allocation_ref(mali_alloc); -+ -+ return 0; -+} -+ -+_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor) -+{ -+ u32 size = descriptor->psize; -+ struct mali_session_data *session = descriptor->session; -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); -+ -+ /* Map dma-buf into this session's page tables */ -+ -+ if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { -+ size += MALI_MMU_PAGE_SIZE; -+ } -+ -+ return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start, size); -+} -+ -+_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size) -+{ -+ u32 old_size = descriptor->psize; -+ struct mali_session_data *session = descriptor->session; -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_ALLOCATION_VALID_MAGIC == descriptor->magic); -+ -+ if (descriptor->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { -+ new_size += MALI_MMU_PAGE_SIZE; -+ } -+ -+ if (new_size > old_size) { -+ MALI_DEBUG_ASSERT(new_size <= descriptor->mali_vma_node.vm_node.size); -+ return mali_mmu_pagedir_map(session->page_directory, descriptor->mali_vma_node.vm_node.start + old_size, new_size - old_size); -+ } -+ return _MALI_OSK_ERR_OK; -+} -+ -+void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags) -+{ -+ if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { -+ size += MALI_MMU_PAGE_SIZE; -+ } -+ -+ /* Umap and flush L2 */ -+ mali_mmu_pagedir_unmap(session->page_directory, vaddr, size); -+ mali_executor_zap_all_active(session); -+} -+ -+u32 _mali_ukk_report_memory_usage(void) -+{ -+ u32 sum = 0; -+ -+ if (MALI_TRUE == mali_memory_have_dedicated_memory()) { -+ sum += mali_mem_block_allocator_stat(); -+ } -+ -+ sum += mali_mem_os_stat(); -+ -+ return sum; -+} -+ -+u32 _mali_ukk_report_total_memory_size(void) -+{ -+ return mali_dedicated_mem_size + mali_shared_mem_size; -+} -+ -+ -+/** -+ * Per-session memory descriptor mapping table sizes -+ */ -+#define MALI_MEM_DESCRIPTORS_INIT 64 -+#define MALI_MEM_DESCRIPTORS_MAX 65536 -+ -+_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session_data) -+{ -+ MALI_DEBUG_PRINT(5, ("Memory session begin\n")); -+ -+ session_data->memory_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, -+ _MALI_OSK_LOCK_ORDER_MEM_SESSION); -+ -+ if (NULL == session_data->memory_lock) { -+ MALI_ERROR(_MALI_OSK_ERR_FAULT); -+ } -+ -+ session_data->cow_lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); -+ if (NULL == session_data->cow_lock) { -+ _mali_osk_mutex_term(session_data->memory_lock); -+ MALI_ERROR(_MALI_OSK_ERR_FAULT); -+ } -+ -+ mali_memory_manager_init(&session_data->allocation_mgr); -+ -+ MALI_DEBUG_PRINT(5, ("MMU session begin: success\n")); -+ MALI_SUCCESS; -+} -+ -+void mali_memory_session_end(struct mali_session_data *session) -+{ -+ MALI_DEBUG_PRINT(3, ("MMU session end\n")); -+ -+ if (NULL == session) { -+ MALI_DEBUG_PRINT(1, ("No session data found during session end\n")); -+ return; -+ } -+ /* free allocation */ -+ mali_free_session_allocations(session); -+ /* do some check in unint*/ -+ mali_memory_manager_uninit(&session->allocation_mgr); -+ -+ /* Free the lock */ -+ _mali_osk_mutex_term(session->memory_lock); -+ _mali_osk_mutex_term(session->cow_lock); -+ return; -+} -+ -+_mali_osk_errcode_t mali_memory_initialize(void) -+{ -+ _mali_osk_errcode_t err; -+ -+ idr_init(&mali_backend_idr); -+ mutex_init(&mali_idr_mutex); -+ -+ err = mali_mem_swap_init(); -+ if (err != _MALI_OSK_ERR_OK) { -+ return err; -+ } -+ err = mali_mem_os_init(); -+ if (_MALI_OSK_ERR_OK == err) { -+ err = mali_mem_defer_bind_manager_init(); -+ } -+ -+ return err; -+} -+ -+void mali_memory_terminate(void) -+{ -+ mali_mem_swap_term(); -+ mali_mem_defer_bind_manager_destory(); -+ mali_mem_os_term(); -+ if (mali_memory_have_dedicated_memory()) { -+ mali_mem_block_allocator_destroy(); -+ } -+} -+ -+ -+struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type) -+{ -+ mali_page_node *page_node = NULL; -+ -+ page_node = kzalloc(sizeof(mali_page_node), GFP_KERNEL); -+ MALI_DEBUG_ASSERT(NULL != page_node); -+ -+ if (page_node) { -+ page_node->type = type; -+ INIT_LIST_HEAD(&page_node->list); -+ } -+ -+ return page_node; -+} -+ -+void _mali_page_node_ref(struct mali_page_node *node) -+{ -+ if (node->type == MALI_PAGE_NODE_OS) { -+ /* add ref to this page */ -+ get_page(node->page); -+ } else if (node->type == MALI_PAGE_NODE_BLOCK) { -+ mali_mem_block_add_ref(node); -+ } else if (node->type == MALI_PAGE_NODE_SWAP) { -+ atomic_inc(&node->swap_it->ref_count); -+ } else { -+ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); -+ } -+} -+ -+void _mali_page_node_unref(struct mali_page_node *node) -+{ -+ if (node->type == MALI_PAGE_NODE_OS) { -+ /* unref to this page */ -+ put_page(node->page); -+ } else if (node->type == MALI_PAGE_NODE_BLOCK) { -+ mali_mem_block_dec_ref(node); -+ } else { -+ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); -+ } -+} -+ -+ -+void _mali_page_node_add_page(struct mali_page_node *node, struct page *page) -+{ -+ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_OS == node->type); -+ node->page = page; -+} -+ -+ -+void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item) -+{ -+ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_SWAP == node->type); -+ node->swap_it = item; -+} -+ -+void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item) -+{ -+ MALI_DEBUG_ASSERT(MALI_PAGE_NODE_BLOCK == node->type); -+ node->blk_it = item; -+} -+ -+ -+int _mali_page_node_get_ref_count(struct mali_page_node *node) -+{ -+ if (node->type == MALI_PAGE_NODE_OS) { -+ /* get ref count of this page */ -+ return page_count(node->page); -+ } else if (node->type == MALI_PAGE_NODE_BLOCK) { -+ return mali_mem_block_get_ref_count(node); -+ } else if (node->type == MALI_PAGE_NODE_SWAP) { -+ return atomic_read(&node->swap_it->ref_count); -+ } else { -+ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); -+ } -+ return -1; -+} -+ -+ -+dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node) -+{ -+ if (node->type == MALI_PAGE_NODE_OS) { -+ return page_private(node->page); -+ } else if (node->type == MALI_PAGE_NODE_BLOCK) { -+ return _mali_blk_item_get_phy_addr(node->blk_it); -+ } else if (node->type == MALI_PAGE_NODE_SWAP) { -+ return node->swap_it->dma_addr; -+ } else { -+ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); -+ } -+ return 0; -+} -+ -+ -+unsigned long _mali_page_node_get_pfn(struct mali_page_node *node) -+{ -+ if (node->type == MALI_PAGE_NODE_OS) { -+ return page_to_pfn(node->page); -+ } else if (node->type == MALI_PAGE_NODE_BLOCK) { -+ /* get phy addr for BLOCK page*/ -+ return _mali_blk_item_get_pfn(node->blk_it); -+ } else if (node->type == MALI_PAGE_NODE_SWAP) { -+ return page_to_pfn(node->swap_it->page); -+ } else { -+ MALI_DEBUG_PRINT_ERROR(("Invalid type of mali page node! \n")); -+ } -+ return 0; -+} -+ -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h -new file mode 100755 -index 000000000..efebbef23 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory.h -@@ -0,0 +1,143 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_H__ -+#define __MALI_MEMORY_H__ -+ -+#include "mali_osk.h" -+#include "mali_session.h" -+ -+#include -+#include -+ -+#include "mali_memory_types.h" -+#include "mali_memory_os_alloc.h" -+ -+_mali_osk_errcode_t mali_memory_initialize(void); -+void mali_memory_terminate(void); -+ -+/** @brief Allocate a page table page -+ * -+ * Allocate a page for use as a page directory or page table. The page is -+ * mapped into kernel space. -+ * -+ * @return _MALI_OSK_ERR_OK on success, otherwise an error code -+ * @param table_page GPU pointer to the allocated page -+ * @param mapping CPU pointer to the mapping of the allocated page -+ */ -+MALI_STATIC_INLINE _mali_osk_errcode_t -+mali_mmu_get_table_page(mali_dma_addr *table_page, mali_io_address *mapping) -+{ -+ return mali_mem_os_get_table_page(table_page, mapping); -+} -+ -+/** @brief Release a page table page -+ * -+ * Release a page table page allocated through \a mali_mmu_get_table_page -+ * -+ * @param pa the GPU address of the page to release -+ */ -+MALI_STATIC_INLINE void -+mali_mmu_release_table_page(mali_dma_addr phys, void *virt) -+{ -+ mali_mem_os_release_table_page(phys, virt); -+} -+ -+/** @brief mmap function -+ * -+ * mmap syscalls on the Mali device node will end up here. -+ * -+ * This function allocates Mali memory and maps it on CPU and Mali. -+ */ -+int mali_mmap(struct file *filp, struct vm_area_struct *vma); -+ -+/** @brief Start a new memory session -+ * -+ * Called when a process opens the Mali device node. -+ * -+ * @param session Pointer to session to initialize -+ */ -+_mali_osk_errcode_t mali_memory_session_begin(struct mali_session_data *session); -+ -+/** @brief Close a memory session -+ * -+ * Called when a process closes the Mali device node. -+ * -+ * Memory allocated by the session will be freed -+ * -+ * @param session Pointer to the session to terminate -+ */ -+void mali_memory_session_end(struct mali_session_data *session); -+ -+/** @brief Prepare Mali page tables for mapping -+ * -+ * This function will prepare the Mali page tables for mapping the memory -+ * described by \a descriptor. -+ * -+ * Page tables will be reference counted and allocated, if not yet present. -+ * -+ * @param descriptor Pointer to the memory descriptor to the mapping -+ */ -+_mali_osk_errcode_t mali_mem_mali_map_prepare(mali_mem_allocation *descriptor); -+ -+/** @brief Resize Mali page tables for mapping -+ * -+ * This function will Resize the Mali page tables for mapping the memory -+ * described by \a descriptor. -+ * -+ * Page tables will be reference counted and allocated, if not yet present. -+ * -+ * @param descriptor Pointer to the memory descriptor to the mapping -+ * @param new_size The new size of descriptor -+ */ -+_mali_osk_errcode_t mali_mem_mali_map_resize(mali_mem_allocation *descriptor, u32 new_size); -+ -+/** @brief Free Mali page tables for mapping -+ * -+ * This function will unmap pages from Mali memory and free the page tables -+ * that are now unused. -+ * -+ * The updated pages in the Mali L2 cache will be invalidated, and the MMU TLBs will be zapped if necessary. -+ * -+ * @param descriptor Pointer to the memory descriptor to unmap -+ */ -+void mali_mem_mali_map_free(struct mali_session_data *session, u32 size, mali_address_t vaddr, u32 flags); -+ -+/** @brief Parse resource and prepare the OS memory allocator -+ * -+ * @param size Maximum size to allocate for Mali GPU. -+ * @return _MALI_OSK_ERR_OK on success, otherwise failure. -+ */ -+_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size); -+ -+/** @brief Parse resource and prepare the dedicated memory allocator -+ * -+ * @param start Physical start address of dedicated Mali GPU memory. -+ * @param size Size of dedicated Mali GPU memory. -+ * @return _MALI_OSK_ERR_OK on success, otherwise failure. -+ */ -+_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); -+ -+ -+struct mali_page_node *_mali_page_node_allocate(mali_page_node_type type); -+ -+void _mali_page_node_ref(struct mali_page_node *node); -+void _mali_page_node_unref(struct mali_page_node *node); -+void _mali_page_node_add_page(struct mali_page_node *node, struct page *page); -+ -+void _mali_page_node_add_block_item(struct mali_page_node *node, mali_block_item *item); -+ -+void _mali_page_node_add_swap_item(struct mali_page_node *node, struct mali_swap_item *item); -+ -+int _mali_page_node_get_ref_count(struct mali_page_node *node); -+dma_addr_t _mali_page_node_get_dma_addr(struct mali_page_node *node); -+unsigned long _mali_page_node_get_pfn(struct mali_page_node *node); -+ -+#endif /* __MALI_MEMORY_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c -new file mode 100755 -index 000000000..bccef3576 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.c -@@ -0,0 +1,362 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_memory.h" -+#include "mali_memory_block_alloc.h" -+#include "mali_osk.h" -+#include -+ -+ -+static mali_block_allocator *mali_mem_block_gobal_allocator = NULL; -+ -+unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item) -+{ -+ return (item->phy_addr & ~(MALI_BLOCK_REF_MASK)); -+} -+ -+ -+unsigned long _mali_blk_item_get_pfn(mali_block_item *item) -+{ -+ return (item->phy_addr / MALI_BLOCK_SIZE); -+} -+ -+ -+u32 mali_mem_block_get_ref_count(mali_page_node *node) -+{ -+ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); -+ return (node->blk_it->phy_addr & MALI_BLOCK_REF_MASK); -+} -+ -+ -+/* Increase the refence count -+* It not atomic, so it need to get sp_lock before call this function -+*/ -+ -+u32 mali_mem_block_add_ref(mali_page_node *node) -+{ -+ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); -+ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) < MALI_BLOCK_MAX_REF_COUNT); -+ return (node->blk_it->phy_addr++ & MALI_BLOCK_REF_MASK); -+} -+ -+/* Decase the refence count -+* It not atomic, so it need to get sp_lock before call this function -+*/ -+u32 mali_mem_block_dec_ref(mali_page_node *node) -+{ -+ MALI_DEBUG_ASSERT(node->type == MALI_PAGE_NODE_BLOCK); -+ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(node) > 0); -+ return (node->blk_it->phy_addr-- & MALI_BLOCK_REF_MASK); -+} -+ -+ -+static mali_block_allocator *mali_mem_block_allocator_create(u32 base_address, u32 size) -+{ -+ mali_block_allocator *info; -+ u32 usable_size; -+ u32 num_blocks; -+ mali_page_node *m_node; -+ mali_block_item *mali_blk_items = NULL; -+ int i = 0; -+ -+ usable_size = size & ~(MALI_BLOCK_SIZE - 1); -+ MALI_DEBUG_PRINT(3, ("Mali block allocator create for region starting at 0x%08X length 0x%08X\n", base_address, size)); -+ MALI_DEBUG_PRINT(4, ("%d usable bytes\n", usable_size)); -+ num_blocks = usable_size / MALI_BLOCK_SIZE; -+ MALI_DEBUG_PRINT(4, ("which becomes %d blocks\n", num_blocks)); -+ -+ if (usable_size == 0) { -+ MALI_DEBUG_PRINT(1, ("Memory block of size %d is unusable\n", size)); -+ return NULL; -+ } -+ -+ info = _mali_osk_calloc(1, sizeof(mali_block_allocator)); -+ if (NULL != info) { -+ INIT_LIST_HEAD(&info->free); -+ spin_lock_init(&info->sp_lock); -+ info->total_num = num_blocks; -+ mali_blk_items = _mali_osk_calloc(1, sizeof(mali_block_item) * num_blocks); -+ -+ if (mali_blk_items) { -+ info->items = mali_blk_items; -+ /* add blocks(4k size) to free list*/ -+ for (i = 0 ; i < num_blocks ; i++) { -+ /* add block information*/ -+ mali_blk_items[i].phy_addr = base_address + (i * MALI_BLOCK_SIZE); -+ /* add to free list */ -+ m_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); -+ if (m_node == NULL) -+ goto fail; -+ _mali_page_node_add_block_item(m_node, &(mali_blk_items[i])); -+ list_add_tail(&m_node->list, &info->free); -+ atomic_add(1, &info->free_num); -+ } -+ return info; -+ } -+ } -+fail: -+ mali_mem_block_allocator_destroy(); -+ return NULL; -+} -+ -+void mali_mem_block_allocator_destroy(void) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ mali_block_allocator *info = mali_mem_block_gobal_allocator; -+ MALI_DEBUG_ASSERT_POINTER(info); -+ MALI_DEBUG_PRINT(4, ("Memory block destroy !\n")); -+ -+ if (NULL == info) -+ return; -+ -+ list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); -+ list_del(&m_page->list); -+ kfree(m_page); -+ } -+ -+ _mali_osk_free(info->items); -+ _mali_osk_free(info); -+} -+ -+u32 mali_mem_block_release(mali_mem_backend *mem_bkend) -+{ -+ mali_mem_allocation *alloc = mem_bkend->mali_allocation; -+ u32 free_pages_nr = 0; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); -+ -+ /* Unmap the memory from the mali virtual address space. */ -+ mali_mem_block_mali_unmap(alloc); -+ mutex_lock(&mem_bkend->mutex); -+ free_pages_nr = mali_mem_block_free(&mem_bkend->block_mem); -+ mutex_unlock(&mem_bkend->mutex); -+ return free_pages_nr; -+} -+ -+ -+int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; -+ mali_block_allocator *info = mali_mem_block_gobal_allocator; -+ MALI_DEBUG_ASSERT_POINTER(info); -+ -+ MALI_DEBUG_PRINT(4, ("BLOCK Mem: Allocate size = 0x%x\n", size)); -+ /*do some init */ -+ INIT_LIST_HEAD(&block_mem->pfns); -+ -+ spin_lock(&info->sp_lock); -+ /*check if have enough space*/ -+ if (atomic_read(&info->free_num) > page_count) { -+ list_for_each_entry_safe(m_page, m_tmp , &info->free, list) { -+ if (page_count > 0) { -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); -+ MALI_DEBUG_ASSERT(mali_mem_block_get_ref_count(m_page) == 0); -+ list_move(&m_page->list, &block_mem->pfns); -+ block_mem->count++; -+ atomic_dec(&info->free_num); -+ _mali_page_node_ref(m_page); -+ } else { -+ break; -+ } -+ page_count--; -+ } -+ } else { -+ /* can't allocate from BLOCK memory*/ -+ spin_unlock(&info->sp_lock); -+ return -1; -+ } -+ -+ spin_unlock(&info->sp_lock); -+ return 0; -+} -+ -+u32 mali_mem_block_free(mali_mem_block_mem *block_mem) -+{ -+ u32 free_pages_nr = 0; -+ -+ free_pages_nr = mali_mem_block_free_list(&block_mem->pfns); -+ MALI_DEBUG_PRINT(4, ("BLOCK Mem free : allocated size = 0x%x, free size = 0x%x\n", block_mem->count * _MALI_OSK_MALI_PAGE_SIZE, -+ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); -+ block_mem->count = 0; -+ MALI_DEBUG_ASSERT(list_empty(&block_mem->pfns)); -+ -+ return free_pages_nr; -+} -+ -+ -+u32 mali_mem_block_free_list(struct list_head *list) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ mali_block_allocator *info = mali_mem_block_gobal_allocator; -+ u32 free_pages_nr = 0; -+ -+ if (info) { -+ spin_lock(&info->sp_lock); -+ list_for_each_entry_safe(m_page, m_tmp , list, list) { -+ if (1 == _mali_page_node_get_ref_count(m_page)) { -+ free_pages_nr++; -+ } -+ mali_mem_block_free_node(m_page); -+ } -+ spin_unlock(&info->sp_lock); -+ } -+ return free_pages_nr; -+} -+ -+/* free the node,*/ -+void mali_mem_block_free_node(struct mali_page_node *node) -+{ -+ mali_block_allocator *info = mali_mem_block_gobal_allocator; -+ -+ /* only handle BLOCK node */ -+ if (node->type == MALI_PAGE_NODE_BLOCK && info) { -+ /*Need to make this atomic?*/ -+ if (1 == _mali_page_node_get_ref_count(node)) { -+ /*Move to free list*/ -+ _mali_page_node_unref(node); -+ list_move_tail(&node->list, &info->free); -+ atomic_add(1, &info->free_num); -+ } else { -+ _mali_page_node_unref(node); -+ list_del(&node->list); -+ kfree(node); -+ } -+ } -+} -+ -+/* unref the node, but not free it */ -+_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node) -+{ -+ mali_block_allocator *info = mali_mem_block_gobal_allocator; -+ mali_page_node *new_node; -+ -+ /* only handle BLOCK node */ -+ if (node->type == MALI_PAGE_NODE_BLOCK && info) { -+ /*Need to make this atomic?*/ -+ if (1 == _mali_page_node_get_ref_count(node)) { -+ /* allocate a new node, Add to free list, keep the old node*/ -+ _mali_page_node_unref(node); -+ new_node = _mali_page_node_allocate(MALI_PAGE_NODE_BLOCK); -+ if (new_node) { -+ memcpy(new_node, node, sizeof(mali_page_node)); -+ list_add(&new_node->list, &info->free); -+ atomic_add(1, &info->free_num); -+ } else -+ return _MALI_OSK_ERR_FAULT; -+ -+ } else { -+ _mali_page_node_unref(node); -+ } -+ } -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props) -+{ -+ struct mali_page_directory *pagedir = session->page_directory; -+ struct mali_page_node *m_page; -+ dma_addr_t phys; -+ u32 virt = vaddr; -+ u32 prop = props; -+ -+ list_for_each_entry(m_page, &block_mem->pfns, list) { -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); -+ phys = _mali_page_node_get_dma_addr(m_page); -+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) -+ /* Verify that the "physical" address is 32-bit and -+ * usable for Mali, when on a system with bus addresses -+ * wider than 32-bit. */ -+ MALI_DEBUG_ASSERT(0 == (phys >> 32)); -+#endif -+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); -+ virt += MALI_MMU_PAGE_SIZE; -+ } -+ -+ return 0; -+} -+ -+void mali_mem_block_mali_unmap(mali_mem_allocation *alloc) -+{ -+ struct mali_session_data *session; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ mali_session_memory_lock(session); -+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ mali_session_memory_unlock(session); -+} -+ -+ -+int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) -+{ -+ int ret; -+ mali_mem_block_mem *block_mem = &mem_bkend->block_mem; -+ unsigned long addr = vma->vm_start; -+ struct mali_page_node *m_page; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_BLOCK); -+ -+ list_for_each_entry(m_page, &block_mem->pfns, list) { -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_BLOCK); -+ ret = vmf_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); -+ -+ if (unlikely(0 != ret)) { -+ return -EFAULT; -+ } -+ addr += _MALI_OSK_MALI_PAGE_SIZE; -+ -+ } -+ -+ return 0; -+} -+ -+ -+_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size) -+{ -+ mali_block_allocator *allocator; -+ -+ /* Do the low level linux operation first */ -+ -+ /* Request ownership of the memory */ -+ if (_MALI_OSK_ERR_OK != _mali_osk_mem_reqregion(start, size, "Dedicated Mali GPU memory")) { -+ MALI_DEBUG_PRINT(1, ("Failed to request memory region for frame buffer (0x%08X - 0x%08X)\n", start, start + size - 1)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ /* Create generic block allocator object to handle it */ -+ allocator = mali_mem_block_allocator_create(start, size); -+ -+ if (NULL == allocator) { -+ MALI_DEBUG_PRINT(1, ("Memory bank registration failed\n")); -+ _mali_osk_mem_unreqregion(start, size); -+ MALI_ERROR(_MALI_OSK_ERR_FAULT); -+ } -+ -+ mali_mem_block_gobal_allocator = (mali_block_allocator *)allocator; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+mali_bool mali_memory_have_dedicated_memory(void) -+{ -+ return mali_mem_block_gobal_allocator ? MALI_TRUE : MALI_FALSE; -+} -+ -+u32 mali_mem_block_allocator_stat(void) -+{ -+ mali_block_allocator *allocator = mali_mem_block_gobal_allocator; -+ MALI_DEBUG_ASSERT_POINTER(allocator); -+ -+ return (allocator->total_num - atomic_read(&allocator->free_num)) * _MALI_OSK_MALI_PAGE_SIZE; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h -new file mode 100755 -index 000000000..70fd9ec25 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_block_alloc.h -@@ -0,0 +1,58 @@ -+/* -+ * Copyright (C) 2010, 2013, 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_BLOCK_ALLOCATOR_H__ -+#define __MALI_BLOCK_ALLOCATOR_H__ -+ -+#include "mali_session.h" -+#include "mali_memory.h" -+#include -+ -+#include "mali_memory_types.h" -+ -+#define MALI_BLOCK_SIZE (PAGE_SIZE) /* 4 kB, manage BLOCK memory as page size */ -+#define MALI_BLOCK_REF_MASK (0xFFF) -+#define MALI_BLOCK_MAX_REF_COUNT (0xFFF) -+ -+ -+ -+typedef struct mali_block_allocator { -+ /* -+ * In free list, each node's ref_count is 0, -+ * ref_count added when allocated or referenced in COW -+ */ -+ mali_block_item *items; /* information for each block item*/ -+ struct list_head free; /*free list of mali_memory_node*/ -+ spinlock_t sp_lock; /*lock for reference count & free list opertion*/ -+ u32 total_num; /* Number of total pages*/ -+ atomic_t free_num; /*number of free pages*/ -+} mali_block_allocator; -+ -+unsigned long _mali_blk_item_get_phy_addr(mali_block_item *item); -+unsigned long _mali_blk_item_get_pfn(mali_block_item *item); -+u32 mali_mem_block_get_ref_count(mali_page_node *node); -+u32 mali_mem_block_add_ref(mali_page_node *node); -+u32 mali_mem_block_dec_ref(mali_page_node *node); -+u32 mali_mem_block_release(mali_mem_backend *mem_bkend); -+int mali_mem_block_alloc(mali_mem_block_mem *block_mem, u32 size); -+int mali_mem_block_mali_map(mali_mem_block_mem *block_mem, struct mali_session_data *session, u32 vaddr, u32 props); -+void mali_mem_block_mali_unmap(mali_mem_allocation *alloc); -+ -+int mali_mem_block_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); -+_mali_osk_errcode_t mali_memory_core_resource_dedicated_memory(u32 start, u32 size); -+mali_bool mali_memory_have_dedicated_memory(void); -+u32 mali_mem_block_free(mali_mem_block_mem *block_mem); -+u32 mali_mem_block_free_list(struct list_head *list); -+void mali_mem_block_free_node(struct mali_page_node *node); -+void mali_mem_block_allocator_destroy(void); -+_mali_osk_errcode_t mali_mem_block_unref_node(struct mali_page_node *node); -+u32 mali_mem_block_allocator_stat(void); -+ -+#endif /* __MALI_BLOCK_ALLOCATOR_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c -new file mode 100755 -index 000000000..0bdf90b16 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.c -@@ -0,0 +1,776 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#ifdef CONFIG_ARM -+#include -+#endif -+#include -+ -+#include "mali_memory.h" -+#include "mali_kernel_common.h" -+#include "mali_uk_types.h" -+#include "mali_osk.h" -+#include "mali_kernel_linux.h" -+#include "mali_memory_cow.h" -+#include "mali_memory_block_alloc.h" -+#include "mali_memory_swap_alloc.h" -+ -+/** -+* allocate pages for COW backend and flush cache -+*/ -+static struct page *mali_mem_cow_alloc_page(void) -+ -+{ -+ mali_mem_os_mem os_mem; -+ struct mali_page_node *node; -+ struct page *new_page; -+ -+ int ret = 0; -+ /* allocate pages from os mem */ -+ ret = mali_mem_os_alloc_pages(&os_mem, _MALI_OSK_MALI_PAGE_SIZE); -+ -+ if (ret) { -+ return NULL; -+ } -+ -+ MALI_DEBUG_ASSERT(1 == os_mem.count); -+ -+ node = _MALI_OSK_CONTAINER_OF(os_mem.pages.next, struct mali_page_node, list); -+ new_page = node->page; -+ node->page = NULL; -+ list_del(&node->list); -+ kfree(node); -+ -+ return new_page; -+} -+ -+ -+static struct list_head *_mali_memory_cow_get_node_list(mali_mem_backend *target_bk, -+ u32 target_offset, -+ u32 target_size) -+{ -+ MALI_DEBUG_ASSERT(MALI_MEM_OS == target_bk->type || MALI_MEM_COW == target_bk->type || -+ MALI_MEM_BLOCK == target_bk->type || MALI_MEM_SWAP == target_bk->type); -+ -+ if (MALI_MEM_OS == target_bk->type) { -+ MALI_DEBUG_ASSERT(&target_bk->os_mem); -+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->os_mem.count); -+ return &target_bk->os_mem.pages; -+ } else if (MALI_MEM_COW == target_bk->type) { -+ MALI_DEBUG_ASSERT(&target_bk->cow_mem); -+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->cow_mem.count); -+ return &target_bk->cow_mem.pages; -+ } else if (MALI_MEM_BLOCK == target_bk->type) { -+ MALI_DEBUG_ASSERT(&target_bk->block_mem); -+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->block_mem.count); -+ return &target_bk->block_mem.pfns; -+ } else if (MALI_MEM_SWAP == target_bk->type) { -+ MALI_DEBUG_ASSERT(&target_bk->swap_mem); -+ MALI_DEBUG_ASSERT(((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE) <= target_bk->swap_mem.count); -+ return &target_bk->swap_mem.pages; -+ } -+ -+ return NULL; -+} -+ -+/** -+* Do COW for os memory - support do COW for memory from bank memory -+* The range_start/size can be zero, which means it will call cow_modify_range -+* latter. -+* This function allocate new pages for COW backend from os mem for a modified range -+* It will keep the page which not in the modified range and Add ref to it -+* -+* @target_bk - target allocation's backend(the allocation need to do COW) -+* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) -+* @target_size - size of target allocation to do COW (for support memory bank) -+* @backend -COW backend -+* @range_start - offset of modified range (4K align) -+* @range_size - size of modified range -+*/ -+_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, -+ u32 target_offset, -+ u32 target_size, -+ mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size) -+{ -+ mali_mem_cow *cow = &backend->cow_mem; -+ struct mali_page_node *m_page, *m_tmp, *page_node; -+ int target_page = 0; -+ struct page *new_page; -+ struct list_head *pages = NULL; -+ -+ pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); -+ -+ if (NULL == pages) { -+ MALI_DEBUG_PRINT_ERROR(("No memory page need to cow ! \n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ MALI_DEBUG_ASSERT(0 == cow->count); -+ -+ INIT_LIST_HEAD(&cow->pages); -+ mutex_lock(&target_bk->mutex); -+ list_for_each_entry_safe(m_page, m_tmp, pages, list) { -+ /* add page from (target_offset,target_offset+size) to cow backend */ -+ if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && -+ (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { -+ -+ /* allocate a new page node, alway use OS memory for COW */ -+ page_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); -+ -+ if (NULL == page_node) { -+ mutex_unlock(&target_bk->mutex); -+ goto error; -+ } -+ -+ INIT_LIST_HEAD(&page_node->list); -+ -+ /* check if in the modified range*/ -+ if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && -+ (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { -+ /* need to allocate a new page */ -+ /* To simplify the case, All COW memory is allocated from os memory ?*/ -+ new_page = mali_mem_cow_alloc_page(); -+ -+ if (NULL == new_page) { -+ kfree(page_node); -+ mutex_unlock(&target_bk->mutex); -+ goto error; -+ } -+ -+ _mali_page_node_add_page(page_node, new_page); -+ } else { -+ /*Add Block memory case*/ -+ if (m_page->type != MALI_PAGE_NODE_BLOCK) { -+ _mali_page_node_add_page(page_node, m_page->page); -+ } else { -+ page_node->type = MALI_PAGE_NODE_BLOCK; -+ _mali_page_node_add_block_item(page_node, m_page->blk_it); -+ } -+ -+ /* add ref to this page */ -+ _mali_page_node_ref(m_page); -+ } -+ -+ /* add it to COW backend page list */ -+ list_add_tail(&page_node->list, &cow->pages); -+ cow->count++; -+ } -+ target_page++; -+ } -+ mutex_unlock(&target_bk->mutex); -+ return _MALI_OSK_ERR_OK; -+error: -+ mali_mem_cow_release(backend, MALI_FALSE); -+ return _MALI_OSK_ERR_FAULT; -+} -+ -+_mali_osk_errcode_t mali_memory_cow_swap_memory(mali_mem_backend *target_bk, -+ u32 target_offset, -+ u32 target_size, -+ mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size) -+{ -+ mali_mem_cow *cow = &backend->cow_mem; -+ struct mali_page_node *m_page, *m_tmp, *page_node; -+ int target_page = 0; -+ struct mali_swap_item *swap_item; -+ struct list_head *pages = NULL; -+ -+ pages = _mali_memory_cow_get_node_list(target_bk, target_offset, target_size); -+ if (NULL == pages) { -+ MALI_DEBUG_PRINT_ERROR(("No swap memory page need to cow ! \n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ MALI_DEBUG_ASSERT(0 == cow->count); -+ -+ INIT_LIST_HEAD(&cow->pages); -+ mutex_lock(&target_bk->mutex); -+ -+ backend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; -+ -+ list_for_each_entry_safe(m_page, m_tmp, pages, list) { -+ /* add page from (target_offset,target_offset+size) to cow backend */ -+ if ((target_page >= target_offset / _MALI_OSK_MALI_PAGE_SIZE) && -+ (target_page < ((target_size + target_offset) / _MALI_OSK_MALI_PAGE_SIZE))) { -+ -+ /* allocate a new page node, use swap memory for COW memory swap cowed flag. */ -+ page_node = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); -+ -+ if (NULL == page_node) { -+ mutex_unlock(&target_bk->mutex); -+ goto error; -+ } -+ -+ /* check if in the modified range*/ -+ if ((cow->count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && -+ (cow->count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { -+ /* need to allocate a new page */ -+ /* To simplify the case, All COW memory is allocated from os memory ?*/ -+ swap_item = mali_mem_swap_alloc_swap_item(); -+ -+ if (NULL == swap_item) { -+ kfree(page_node); -+ mutex_unlock(&target_bk->mutex); -+ goto error; -+ } -+ -+ swap_item->idx = mali_mem_swap_idx_alloc(); -+ -+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { -+ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW.\n")); -+ kfree(page_node); -+ kfree(swap_item); -+ mutex_unlock(&target_bk->mutex); -+ goto error; -+ } -+ -+ _mali_page_node_add_swap_item(page_node, swap_item); -+ } else { -+ _mali_page_node_add_swap_item(page_node, m_page->swap_it); -+ -+ /* add ref to this page */ -+ _mali_page_node_ref(m_page); -+ } -+ -+ list_add_tail(&page_node->list, &cow->pages); -+ cow->count++; -+ } -+ target_page++; -+ } -+ mutex_unlock(&target_bk->mutex); -+ -+ return _MALI_OSK_ERR_OK; -+error: -+ mali_mem_swap_release(backend, MALI_FALSE); -+ return _MALI_OSK_ERR_FAULT; -+ -+} -+ -+ -+_mali_osk_errcode_t _mali_mem_put_page_node(mali_page_node *node) -+{ -+ if (node->type == MALI_PAGE_NODE_OS) { -+ return mali_mem_os_put_page(node->page); -+ } else if (node->type == MALI_PAGE_NODE_BLOCK) { -+ return mali_mem_block_unref_node(node); -+ } else if (node->type == MALI_PAGE_NODE_SWAP) { -+ return _mali_mem_swap_put_page_node(node); -+ } else -+ MALI_DEBUG_ASSERT(0); -+ return _MALI_OSK_ERR_FAULT; -+} -+ -+ -+/** -+* Modify a range of a exist COW backend -+* @backend -COW backend -+* @range_start - offset of modified range (4K align) -+* @range_size - size of modified range(in byte) -+*/ -+_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size) -+{ -+ mali_mem_allocation *alloc = NULL; -+ struct mali_session_data *session; -+ mali_mem_cow *cow = &backend->cow_mem; -+ struct mali_page_node *m_page, *m_tmp; -+ LIST_HEAD(pages); -+ struct page *new_page; -+ u32 count = 0; -+ s32 change_pages_nr = 0; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; -+ -+ if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ -+ alloc = backend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); -+ MALI_DEBUG_ASSERT(((range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE) <= cow->count); -+ -+ mutex_lock(&backend->mutex); -+ -+ /* free pages*/ -+ list_for_each_entry_safe(m_page, m_tmp, &cow->pages, list) { -+ -+ /* check if in the modified range*/ -+ if ((count >= range_start / _MALI_OSK_MALI_PAGE_SIZE) && -+ (count < (range_start + range_size) / _MALI_OSK_MALI_PAGE_SIZE)) { -+ if (MALI_PAGE_NODE_SWAP != m_page->type) { -+ new_page = mali_mem_cow_alloc_page(); -+ -+ if (NULL == new_page) { -+ goto error; -+ } -+ if (1 != _mali_page_node_get_ref_count(m_page)) -+ change_pages_nr++; -+ /* unref old page*/ -+ _mali_osk_mutex_wait(session->cow_lock); -+ if (_mali_mem_put_page_node(m_page)) { -+ __free_page(new_page); -+ _mali_osk_mutex_signal(session->cow_lock); -+ goto error; -+ } -+ _mali_osk_mutex_signal(session->cow_lock); -+ /* add new page*/ -+ /* always use OS for COW*/ -+ m_page->type = MALI_PAGE_NODE_OS; -+ _mali_page_node_add_page(m_page, new_page); -+ } else { -+ struct mali_swap_item *swap_item; -+ -+ swap_item = mali_mem_swap_alloc_swap_item(); -+ -+ if (NULL == swap_item) { -+ goto error; -+ } -+ -+ swap_item->idx = mali_mem_swap_idx_alloc(); -+ -+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == swap_item->idx) { -+ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW modify range.\n")); -+ kfree(swap_item); -+ goto error; -+ } -+ -+ if (1 != _mali_page_node_get_ref_count(m_page)) { -+ change_pages_nr++; -+ } -+ -+ if (_mali_mem_put_page_node(m_page)) { -+ mali_mem_swap_free_swap_item(swap_item); -+ goto error; -+ } -+ -+ _mali_page_node_add_swap_item(m_page, swap_item); -+ } -+ } -+ count++; -+ } -+ cow->change_pages_nr = change_pages_nr; -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == alloc->type); -+ -+ /* ZAP cpu mapping(modified range), and do cpu mapping here if need */ -+ if (NULL != alloc->cpu_mapping.vma) { -+ MALI_DEBUG_ASSERT(0 != alloc->backend_handle); -+ MALI_DEBUG_ASSERT(NULL != alloc->cpu_mapping.vma); -+ MALI_DEBUG_ASSERT(alloc->cpu_mapping.vma->vm_end - alloc->cpu_mapping.vma->vm_start >= range_size); -+ -+ if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { -+ zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); -+ -+ ret = mali_mem_cow_cpu_map_pages_locked(backend, alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size / _MALI_OSK_MALI_PAGE_SIZE); -+ -+ if (unlikely(ret != _MALI_OSK_ERR_OK)) { -+ MALI_DEBUG_PRINT(2, ("mali_memory_cow_modify_range: cpu mapping failed !\n")); -+ ret = _MALI_OSK_ERR_FAULT; -+ } -+ } else { -+ /* used to trigger page fault for swappable cowed memory. */ -+ alloc->cpu_mapping.vma->vm_flags |= VM_PFNMAP; -+ alloc->cpu_mapping.vma->vm_flags |= VM_MIXEDMAP; -+ -+ zap_vma_ptes(alloc->cpu_mapping.vma, alloc->cpu_mapping.vma->vm_start + range_start, range_size); -+ /* delete this flag to let swappble is ummapped regard to stauct page not page frame. */ -+ alloc->cpu_mapping.vma->vm_flags &= ~VM_PFNMAP; -+ alloc->cpu_mapping.vma->vm_flags &= ~VM_MIXEDMAP; -+ } -+ } -+ -+error: -+ mutex_unlock(&backend->mutex); -+ return ret; -+ -+} -+ -+ -+/** -+* Allocate pages for COW backend -+* @alloc -allocation for COW allocation -+* @target_bk - target allocation's backend(the allocation need to do COW) -+* @target_offset - the offset in target allocation to do COW(for support COW a memory allocated from memory_bank, 4K align) -+* @target_size - size of target allocation to do COW (for support memory bank)(in byte) -+* @backend -COW backend -+* @range_start - offset of modified range (4K align) -+* @range_size - size of modified range(in byte) -+*/ -+_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, -+ u32 target_offset, -+ u32 target_size, -+ mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size) -+{ -+ struct mali_session_data *session = backend->mali_allocation->session; -+ -+ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); -+ -+ /* size & offset must be a multiple of the system page size */ -+ if (target_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ if (range_size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ if (target_offset % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ if (range_start % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ -+ /* check backend type */ -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == backend->type); -+ -+ switch (target_bk->type) { -+ case MALI_MEM_OS: -+ case MALI_MEM_BLOCK: -+ return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); -+ break; -+ case MALI_MEM_COW: -+ if (backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { -+ return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); -+ } else { -+ return mali_memory_cow_os_memory(target_bk, target_offset, target_size, backend, range_start, range_size); -+ } -+ break; -+ case MALI_MEM_SWAP: -+ return mali_memory_cow_swap_memory(target_bk, target_offset, target_size, backend, range_start, range_size); -+ break; -+ case MALI_MEM_EXTERNAL: -+ /*NOT support yet*/ -+ MALI_DEBUG_PRINT_ERROR(("External physical memory not supported ! \n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ break; -+ case MALI_MEM_DMA_BUF: -+ /*NOT support yet*/ -+ MALI_DEBUG_PRINT_ERROR(("DMA buffer not supported ! \n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ break; -+ case MALI_MEM_UMP: -+ /*NOT support yet*/ -+ MALI_DEBUG_PRINT_ERROR(("UMP buffer not supported ! \n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ break; -+ default: -+ /*Not support yet*/ -+ MALI_DEBUG_PRINT_ERROR(("Invalid memory type not supported ! \n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ break; -+ } -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+/** -+* Map COW backend memory to mali -+* Support OS/BLOCK for mali_page_node -+*/ -+int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size) -+{ -+ mali_mem_allocation *cow_alloc; -+ struct mali_page_node *m_page; -+ struct mali_session_data *session; -+ struct mali_page_directory *pagedir; -+ u32 virt, start; -+ -+ cow_alloc = mem_bkend->mali_allocation; -+ virt = cow_alloc->mali_vma_node.vm_node.start; -+ start = virt; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend); -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); -+ MALI_DEBUG_ASSERT_POINTER(cow_alloc); -+ -+ session = cow_alloc->session; -+ pagedir = session->page_directory; -+ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); -+ list_for_each_entry(m_page, &mem_bkend->cow_mem.pages, list) { -+ if ((virt - start >= range_start) && (virt - start < range_start + range_size)) { -+ dma_addr_t phys = _mali_page_node_get_dma_addr(m_page); -+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) -+ MALI_DEBUG_ASSERT(0 == (phys >> 32)); -+#endif -+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, -+ MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); -+ } -+ virt += MALI_MMU_PAGE_SIZE; -+ } -+ return 0; -+} -+ -+/** -+* Map COW backend to cpu -+* support OS/BLOCK memory -+*/ -+int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) -+{ -+ mali_mem_cow *cow = &mem_bkend->cow_mem; -+ struct mali_page_node *m_page; -+ int ret; -+ unsigned long addr = vma->vm_start; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); -+ -+ list_for_each_entry(m_page, &cow->pages, list) { -+ /* We should use vm_insert_page, but it does a dcache -+ * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn. -+ ret = vm_insert_page(vma, addr, page); -+ */ -+ ret = vmf_insert_pfn(vma, addr, _mali_page_node_get_pfn(m_page)); -+ -+ if (unlikely(0 != ret)) { -+ return ret; -+ } -+ addr += _MALI_OSK_MALI_PAGE_SIZE; -+ } -+ -+ return 0; -+} -+ -+/** -+* Map some pages(COW backend) to CPU vma@vaddr -+*@ mem_bkend - COW backend -+*@ vma -+*@ vaddr -start CPU vaddr mapped to -+*@ num - max number of pages to map to CPU vaddr -+*/ -+_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, -+ struct vm_area_struct *vma, -+ unsigned long vaddr, -+ int num) -+{ -+ mali_mem_cow *cow = &mem_bkend->cow_mem; -+ struct mali_page_node *m_page; -+ int ret; -+ int offset; -+ int count ; -+ unsigned long vstart = vma->vm_start; -+ count = 0; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_COW); -+ MALI_DEBUG_ASSERT(0 == vaddr % _MALI_OSK_MALI_PAGE_SIZE); -+ MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); -+ offset = (vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; -+ -+ list_for_each_entry(m_page, &cow->pages, list) { -+ if ((count >= offset) && (count < offset + num)) { -+ ret = vmf_insert_pfn(vma, vaddr, _mali_page_node_get_pfn(m_page)); -+ -+ if (unlikely(0 != ret)) { -+ if (count == offset) { -+ return _MALI_OSK_ERR_FAULT; -+ } else { -+ /* ret is EBUSY when page isn't in modify range, but now it's OK*/ -+ return _MALI_OSK_ERR_OK; -+ } -+ } -+ vaddr += _MALI_OSK_MALI_PAGE_SIZE; -+ } -+ count++; -+ } -+ return _MALI_OSK_ERR_OK; -+} -+ -+/** -+* Release COW backend memory -+* free it directly(put_page--unref page), not put into pool -+*/ -+u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) -+{ -+ mali_mem_allocation *alloc; -+ struct mali_session_data *session; -+ u32 free_pages_nr = 0; -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend); -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); -+ alloc = mem_bkend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (MALI_MEM_BACKEND_FLAG_SWAP_COWED != (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)) { -+ /* Unmap the memory from the mali virtual address space. */ -+ if (MALI_TRUE == is_mali_mapped) -+ mali_mem_os_mali_unmap(alloc); -+ /* free cow backend list*/ -+ _mali_osk_mutex_wait(session->cow_lock); -+ free_pages_nr = mali_mem_os_free(&mem_bkend->cow_mem.pages, mem_bkend->cow_mem.count, MALI_TRUE); -+ _mali_osk_mutex_signal(session->cow_lock); -+ -+ free_pages_nr += mali_mem_block_free_list(&mem_bkend->cow_mem.pages); -+ -+ MALI_DEBUG_ASSERT(list_empty(&mem_bkend->cow_mem.pages)); -+ } else { -+ free_pages_nr = mali_mem_swap_release(mem_bkend, is_mali_mapped); -+ } -+ -+ -+ MALI_DEBUG_PRINT(4, ("COW Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->cow_mem.count * _MALI_OSK_MALI_PAGE_SIZE, -+ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); -+ -+ mem_bkend->cow_mem.count = 0; -+ return free_pages_nr; -+} -+ -+ -+/* Dst node could os node or swap node. */ -+void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node) -+{ -+ void *dst, *src; -+ struct page *dst_page; -+ dma_addr_t dma_addr; -+ -+ MALI_DEBUG_ASSERT(src_node != NULL); -+ MALI_DEBUG_ASSERT(dst_node != NULL); -+ MALI_DEBUG_ASSERT(dst_node->type == MALI_PAGE_NODE_OS -+ || dst_node->type == MALI_PAGE_NODE_SWAP); -+ -+ if (dst_node->type == MALI_PAGE_NODE_OS) { -+ dst_page = dst_node->page; -+ } else { -+ dst_page = dst_node->swap_it->page; -+ } -+ -+ dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(dst_node), -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ -+ /* map it , and copy the content*/ -+ dst = kmap_atomic(dst_page); -+ -+ if (src_node->type == MALI_PAGE_NODE_OS || -+ src_node->type == MALI_PAGE_NODE_SWAP) { -+ struct page *src_page; -+ -+ if (src_node->type == MALI_PAGE_NODE_OS) { -+ src_page = src_node->page; -+ } else { -+ src_page = src_node->swap_it->page; -+ } -+ -+ /* Clear and invaliate cache */ -+ /* In ARM architecture, speculative read may pull stale data into L1 cache -+ * for kernel linear mapping page table. DMA_BIDIRECTIONAL could -+ * invalidate the L1 cache so that following read get the latest data -+ */ -+ dma_unmap_page(&mali_platform_device->dev, _mali_page_node_get_dma_addr(src_node), -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ -+ src = kmap_atomic(src_page); -+ memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); -+ kunmap_atomic(src); -+ dma_addr = dma_map_page(&mali_platform_device->dev, src_page, -+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ -+ if (src_node->type == MALI_PAGE_NODE_SWAP) { -+ src_node->swap_it->dma_addr = dma_addr; -+ } -+ } else if (src_node->type == MALI_PAGE_NODE_BLOCK) { -+ /* -+ * use ioremap to map src for BLOCK memory -+ */ -+ src = ioremap(_mali_page_node_get_dma_addr(src_node), _MALI_OSK_MALI_PAGE_SIZE); -+ memcpy(dst, src , _MALI_OSK_MALI_PAGE_SIZE); -+ iounmap(src); -+ } -+ kunmap_atomic(dst); -+ dma_addr = dma_map_page(&mali_platform_device->dev, dst_page, -+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ -+ if (dst_node->type == MALI_PAGE_NODE_SWAP) { -+ dst_node->swap_it->dma_addr = dma_addr; -+ } -+} -+ -+ -+/* -+* allocate page on demand when CPU access it, -+* THis used in page fault handler -+*/ -+_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page) -+{ -+ struct page *new_page = NULL; -+ struct mali_page_node *new_node = NULL; -+ int i = 0; -+ struct mali_page_node *m_page, *found_node = NULL; -+ struct mali_session_data *session = NULL; -+ mali_mem_cow *cow = &mem_bkend->cow_mem; -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); -+ MALI_DEBUG_ASSERT(offset_page < mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE); -+ MALI_DEBUG_PRINT(4, ("mali_mem_cow_allocate_on_demand !, offset_page =0x%x\n", offset_page)); -+ -+ /* allocate new page here */ -+ new_page = mali_mem_cow_alloc_page(); -+ if (!new_page) -+ return _MALI_OSK_ERR_NOMEM; -+ -+ new_node = _mali_page_node_allocate(MALI_PAGE_NODE_OS); -+ if (!new_node) { -+ __free_page(new_page); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ /* find the page in backend*/ -+ list_for_each_entry(m_page, &cow->pages, list) { -+ if (i == offset_page) { -+ found_node = m_page; -+ break; -+ } -+ i++; -+ } -+ MALI_DEBUG_ASSERT(found_node); -+ if (NULL == found_node) { -+ __free_page(new_page); -+ kfree(new_node); -+ return _MALI_OSK_ERR_ITEM_NOT_FOUND; -+ } -+ -+ _mali_page_node_add_page(new_node, new_page); -+ -+ /* Copy the src page's content to new page */ -+ _mali_mem_cow_copy_page(found_node, new_node); -+ -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend->mali_allocation); -+ session = mem_bkend->mali_allocation->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ if (1 != _mali_page_node_get_ref_count(found_node)) { -+ atomic_add(1, &session->mali_mem_allocated_pages); -+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { -+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ mem_bkend->cow_mem.change_pages_nr++; -+ } -+ -+ _mali_osk_mutex_wait(session->cow_lock); -+ if (_mali_mem_put_page_node(found_node)) { -+ __free_page(new_page); -+ kfree(new_node); -+ _mali_osk_mutex_signal(session->cow_lock); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ _mali_osk_mutex_signal(session->cow_lock); -+ -+ list_replace(&found_node->list, &new_node->list); -+ -+ kfree(found_node); -+ -+ /* map to GPU side*/ -+ _mali_osk_mutex_wait(session->memory_lock); -+ mali_mem_cow_mali_map(mem_bkend, offset_page * _MALI_OSK_MALI_PAGE_SIZE, _MALI_OSK_MALI_PAGE_SIZE); -+ _mali_osk_mutex_signal(session->memory_lock); -+ return _MALI_OSK_ERR_OK; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h -new file mode 100755 -index 000000000..5f83a37fc ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_cow.h -@@ -0,0 +1,48 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_COW_H__ -+#define __MALI_MEMORY_COW_H__ -+ -+#include "mali_osk.h" -+#include "mali_session.h" -+#include "mali_memory_types.h" -+ -+int mali_mem_cow_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); -+_mali_osk_errcode_t mali_mem_cow_cpu_map_pages_locked(mali_mem_backend *mem_bkend, -+ struct vm_area_struct *vma, -+ unsigned long vaddr, -+ int num); -+ -+_mali_osk_errcode_t mali_memory_do_cow(mali_mem_backend *target_bk, -+ u32 target_offset, -+ u32 target_size, -+ mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size); -+ -+_mali_osk_errcode_t mali_memory_cow_modify_range(mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size); -+ -+_mali_osk_errcode_t mali_memory_cow_os_memory(mali_mem_backend *target_bk, -+ u32 target_offset, -+ u32 target_size, -+ mali_mem_backend *backend, -+ u32 range_start, -+ u32 range_size); -+ -+void _mali_mem_cow_copy_page(mali_page_node *src_node, mali_page_node *dst_node); -+ -+int mali_mem_cow_mali_map(mali_mem_backend *mem_bkend, u32 range_start, u32 range_size); -+u32 mali_mem_cow_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); -+_mali_osk_errcode_t mali_mem_cow_allocate_on_demand(mali_mem_backend *mem_bkend, u32 offset_page); -+#endif -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c -new file mode 100755 -index 000000000..a9db577cb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.c -@@ -0,0 +1,262 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#ifdef CONFIG_ARM -+#include -+#endif -+#include -+ -+#include "mali_memory.h" -+#include "mali_kernel_common.h" -+#include "mali_uk_types.h" -+#include "mali_osk.h" -+#include "mali_kernel_linux.h" -+#include "mali_memory_defer_bind.h" -+#include "mali_executor.h" -+#include "mali_osk.h" -+#include "mali_scheduler.h" -+#include "mali_gp_job.h" -+ -+mali_defer_bind_manager *mali_dmem_man = NULL; -+ -+static u32 mali_dmem_get_gp_varying_size(struct mali_gp_job *gp_job) -+{ -+ return gp_job->required_varying_memsize / _MALI_OSK_MALI_PAGE_SIZE; -+} -+ -+_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void) -+{ -+ mali_dmem_man = _mali_osk_calloc(1, sizeof(struct mali_defer_bind_manager)); -+ if (!mali_dmem_man) -+ return _MALI_OSK_ERR_NOMEM; -+ -+ atomic_set(&mali_dmem_man->num_used_pages, 0); -+ atomic_set(&mali_dmem_man->num_dmem, 0); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+void mali_mem_defer_bind_manager_destory(void) -+{ -+ if (mali_dmem_man) { -+ MALI_DEBUG_ASSERT(0 == atomic_read(&mali_dmem_man->num_dmem)); -+ kfree(mali_dmem_man); -+ } -+ mali_dmem_man = NULL; -+} -+ -+ -+/*allocate pages from OS memory*/ -+_mali_osk_errcode_t mali_mem_defer_alloc_mem(u32 require, struct mali_session_data *session, mali_defer_mem_block *dblock) -+{ -+ int retval = 0; -+ u32 num_pages = require; -+ mali_mem_os_mem os_mem; -+ -+ retval = mali_mem_os_alloc_pages(&os_mem, num_pages * _MALI_OSK_MALI_PAGE_SIZE); -+ -+ /* add to free pages list */ -+ if (0 == retval) { -+ MALI_DEBUG_PRINT(4, ("mali_mem_defer_alloc_mem ,,*** pages allocate = 0x%x \n", num_pages)); -+ list_splice(&os_mem.pages, &dblock->free_pages); -+ atomic_add(os_mem.count, &dblock->num_free_pages); -+ atomic_add(os_mem.count, &session->mali_mem_allocated_pages); -+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { -+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ return _MALI_OSK_ERR_OK; -+ } else -+ return _MALI_OSK_ERR_FAULT; -+} -+ -+_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock) -+{ -+ u32 require_page; -+ -+ if (!next_gp_job) -+ return _MALI_OSK_ERR_FAULT; -+ -+ require_page = mali_dmem_get_gp_varying_size(next_gp_job); -+ -+ MALI_DEBUG_PRINT(4, ("mali_mem_defer_prepare_mem_work, require alloc page 0x%x\n", -+ require_page)); -+ /* allocate more pages from OS */ -+ if (_MALI_OSK_ERR_OK != mali_mem_defer_alloc_mem(require_page, next_gp_job->session, dblock)) { -+ MALI_DEBUG_PRINT(1, ("ERROR##mali_mem_defer_prepare_mem_work, allocate page failed!!")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ next_gp_job->bind_flag = MALI_DEFER_BIND_MEMORY_PREPARED; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+/* do preparetion for allocation before defer bind */ -+_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize) -+{ -+ mali_mem_backend *mem_bkend = NULL; -+ struct mali_backend_bind_list *bk_list = _mali_osk_calloc(1, sizeof(struct mali_backend_bind_list)); -+ if (NULL == bk_list) -+ return _MALI_OSK_ERR_FAULT; -+ -+ INIT_LIST_HEAD(&bk_list->node); -+ /* Get backend memory */ -+ mutex_lock(&mali_idr_mutex); -+ if (!(mem_bkend = idr_find(&mali_backend_idr, alloc->backend_handle))) { -+ MALI_DEBUG_PRINT(1, ("Can't find memory backend in defer bind!\n")); -+ mutex_unlock(&mali_idr_mutex); -+ _mali_osk_free(bk_list); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ mutex_unlock(&mali_idr_mutex); -+ -+ /* If the mem backend has already been bound, no need to bind again.*/ -+ if (mem_bkend->os_mem.count > 0) { -+ _mali_osk_free(bk_list); -+ return _MALI_OSK_ERR_OK; -+ } -+ -+ MALI_DEBUG_PRINT(4, ("bind_allocation_prepare:: allocation =%x vaddr=0x%x!\n", alloc, alloc->mali_vma_node.vm_node.start)); -+ -+ INIT_LIST_HEAD(&mem_bkend->os_mem.pages); -+ -+ bk_list->bkend = mem_bkend; -+ bk_list->vaddr = alloc->mali_vma_node.vm_node.start; -+ bk_list->session = alloc->session; -+ bk_list->page_num = mem_bkend->size / _MALI_OSK_MALI_PAGE_SIZE; -+ *required_varying_memsize += mem_bkend->size; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); -+ -+ /* add to job to do list */ -+ list_add(&bk_list->node, list); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+ -+/* bind phyiscal memory to allocation -+This function will be called in IRQ handler*/ -+static _mali_osk_errcode_t mali_mem_defer_bind_allocation(struct mali_backend_bind_list *bk_node, -+ struct list_head *pages) -+{ -+ struct mali_session_data *session = bk_node->session; -+ mali_mem_backend *mem_bkend = bk_node->bkend; -+ MALI_DEBUG_PRINT(4, ("mali_mem_defer_bind_allocation, bind bkend = %x page num=0x%x vaddr=%x session=%x\n", mem_bkend, bk_node->page_num, bk_node->vaddr, session)); -+ -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); -+ list_splice(pages, &mem_bkend->os_mem.pages); -+ mem_bkend->os_mem.count = bk_node->page_num; -+ -+ if (mem_bkend->type == MALI_MEM_OS) { -+ mali_mem_os_mali_map(&mem_bkend->os_mem, session, bk_node->vaddr, 0, -+ mem_bkend->os_mem.count, MALI_MMU_FLAGS_DEFAULT); -+ } -+ smp_wmb(); -+ bk_node->flag = MALI_DEFER_BIND_MEMORY_BINDED; -+ mem_bkend->flags &= ~MALI_MEM_BACKEND_FLAG_NOT_BINDED; -+ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_BINDED; -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+static struct list_head *mali_mem_defer_get_free_page_list(u32 count, struct list_head *pages, mali_defer_mem_block *dblock) -+{ -+ int i = 0; -+ struct mali_page_node *m_page, *m_tmp; -+ -+ if (atomic_read(&dblock->num_free_pages) < count) { -+ return NULL; -+ } else { -+ list_for_each_entry_safe(m_page, m_tmp, &dblock->free_pages, list) { -+ if (i < count) { -+ list_move_tail(&m_page->list, pages); -+ } else { -+ break; -+ } -+ i++; -+ } -+ MALI_DEBUG_ASSERT(i == count); -+ atomic_sub(count, &dblock->num_free_pages); -+ return pages; -+ } -+} -+ -+ -+/* called in job start IOCTL to bind physical memory for each allocations -+@ bk_list backend list to do defer bind -+@ pages page list to do this bind -+@ count number of pages -+*/ -+_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp, -+ struct mali_defer_mem_block *dmem_block) -+{ -+ struct mali_defer_mem *dmem = NULL; -+ struct mali_backend_bind_list *bkn, *bkn_tmp; -+ LIST_HEAD(pages); -+ -+ if (gp->required_varying_memsize != (atomic_read(&dmem_block->num_free_pages) * _MALI_OSK_MALI_PAGE_SIZE)) { -+ MALI_DEBUG_PRINT_ERROR(("#BIND: The memsize of varying buffer not match to the pagesize of the dmem_block!!## \n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ MALI_DEBUG_PRINT(4, ("#BIND: GP job=%x## \n", gp)); -+ dmem = (mali_defer_mem *)_mali_osk_calloc(1, sizeof(struct mali_defer_mem)); -+ if (dmem) { -+ INIT_LIST_HEAD(&dmem->node); -+ gp->dmem = dmem; -+ } else { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ atomic_add(1, &mali_dmem_man->num_dmem); -+ /* for each bk_list backend, do bind */ -+ list_for_each_entry_safe(bkn, bkn_tmp , &gp->vary_todo, node) { -+ INIT_LIST_HEAD(&pages); -+ if (likely(mali_mem_defer_get_free_page_list(bkn->page_num, &pages, dmem_block))) { -+ list_del(&bkn->node); -+ mali_mem_defer_bind_allocation(bkn, &pages); -+ _mali_osk_free(bkn); -+ } else { -+ /* not enough memory will not happen */ -+ MALI_DEBUG_PRINT_ERROR(("#BIND: NOT enough memory when binded !!## \n")); -+ _mali_osk_free(gp->dmem); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ } -+ -+ if (!list_empty(&gp->vary_todo)) { -+ MALI_DEBUG_PRINT_ERROR(("#BIND: The deferbind backend list isn't empty !!## \n")); -+ _mali_osk_free(gp->dmem); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ dmem->flag = MALI_DEFER_BIND_MEMORY_BINDED; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void mali_mem_defer_dmem_free(struct mali_gp_job *gp) -+{ -+ if (gp->dmem) { -+ atomic_dec(&mali_dmem_man->num_dmem); -+ _mali_osk_free(gp->dmem); -+ } -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h -new file mode 100755 -index 000000000..defa08d52 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_defer_bind.h -@@ -0,0 +1,64 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#ifndef __MALI_MEMORY_DEFER_BIND_H_ -+#define __MALI_MEMORY_DEFER_BIND_H_ -+ -+ -+#include "mali_osk.h" -+#include "mali_session.h" -+ -+#include -+#include -+#include -+#include -+#include -+ -+ -+#include "mali_memory_types.h" -+#include "mali_memory_os_alloc.h" -+#include "mali_uk_types.h" -+ -+struct mali_gp_job; -+ -+typedef struct mali_defer_mem { -+ struct list_head node; /*dlist node in bind manager */ -+ u32 flag; -+} mali_defer_mem; -+ -+ -+typedef struct mali_defer_mem_block { -+ struct list_head free_pages; /* page pool */ -+ atomic_t num_free_pages; -+} mali_defer_mem_block; -+ -+/* varying memory list need to bind */ -+typedef struct mali_backend_bind_list { -+ struct list_head node; -+ struct mali_mem_backend *bkend; -+ u32 vaddr; -+ u32 page_num; -+ struct mali_session_data *session; -+ u32 flag; -+} mali_backend_bind_lists; -+ -+ -+typedef struct mali_defer_bind_manager { -+ atomic_t num_used_pages; -+ atomic_t num_dmem; -+} mali_defer_bind_manager; -+ -+_mali_osk_errcode_t mali_mem_defer_bind_manager_init(void); -+void mali_mem_defer_bind_manager_destory(void); -+_mali_osk_errcode_t mali_mem_defer_bind(struct mali_gp_job *gp, struct mali_defer_mem_block *dmem_block); -+_mali_osk_errcode_t mali_mem_defer_bind_allocation_prepare(mali_mem_allocation *alloc, struct list_head *list, u32 *required_varying_memsize); -+_mali_osk_errcode_t mali_mem_prepare_mem_for_job(struct mali_gp_job *next_gp_job, mali_defer_mem_block *dblock); -+void mali_mem_defer_dmem_free(struct mali_gp_job *gp); -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c -new file mode 100755 -index 000000000..1f4565127 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.c -@@ -0,0 +1,369 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include /* file system operations */ -+#include /* user space access */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_kernel_linux.h" -+ -+#include "mali_memory.h" -+#include "mali_memory_dma_buf.h" -+#include "mali_memory_virtual.h" -+#include "mali_pp_job.h" -+ -+/* -+ * Map DMA buf attachment \a mem into \a session at virtual address \a virt. -+ */ -+static int mali_dma_buf_map(mali_mem_backend *mem_backend) -+{ -+ mali_mem_allocation *alloc; -+ struct mali_dma_buf_attachment *mem; -+ struct mali_session_data *session; -+ struct mali_page_directory *pagedir; -+ _mali_osk_errcode_t err; -+ struct scatterlist *sg; -+ u32 virt, flags; -+ int i; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ -+ alloc = mem_backend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ mem = mem_backend->dma_buf.attachment; -+ MALI_DEBUG_ASSERT_POINTER(mem); -+ -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_ASSERT(mem->session == session); -+ -+ virt = alloc->mali_vma_node.vm_node.start; -+ flags = alloc->flags; -+ -+ mali_session_memory_lock(session); -+ mem->map_ref++; -+ -+ MALI_DEBUG_PRINT(5, ("Mali DMA-buf: map attachment %p, new map_ref = %d\n", mem, mem->map_ref)); -+ -+ if (1 == mem->map_ref) { -+ -+ /* First reference taken, so we need to map the dma buf */ -+ MALI_DEBUG_ASSERT(!mem->is_mapped); -+ -+ mem->sgt = dma_buf_map_attachment(mem->attachment, DMA_BIDIRECTIONAL); -+ if (IS_ERR_OR_NULL(mem->sgt)) { -+ MALI_DEBUG_PRINT_ERROR(("Failed to map dma-buf attachment\n")); -+ mem->map_ref--; -+ mali_session_memory_unlock(session); -+ return -EFAULT; -+ } -+ -+ err = mali_mem_mali_map_prepare(alloc); -+ if (_MALI_OSK_ERR_OK != err) { -+ MALI_DEBUG_PRINT(1, ("Mapping of DMA memory failed\n")); -+ mem->map_ref--; -+ mali_session_memory_unlock(session); -+ return -ENOMEM; -+ } -+ -+ pagedir = mali_session_get_page_directory(session); -+ MALI_DEBUG_ASSERT_POINTER(pagedir); -+ -+ for_each_sg(mem->sgt->sgl, sg, mem->sgt->nents, i) { -+ u32 size = sg_dma_len(sg); -+ dma_addr_t phys = sg_dma_address(sg); -+ -+ /* sg must be page aligned. */ -+ MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); -+ MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); -+ -+ mali_mmu_pagedir_update(pagedir, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); -+ -+ virt += size; -+ } -+ -+ if (flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { -+ u32 guard_phys; -+ MALI_DEBUG_PRINT(7, ("Mapping in extra guard page\n")); -+ -+ guard_phys = sg_dma_address(mem->sgt->sgl); -+ mali_mmu_pagedir_update(pagedir, virt, guard_phys, MALI_MMU_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); -+ } -+ -+ mem->is_mapped = MALI_TRUE; -+ mali_session_memory_unlock(session); -+ /* Wake up any thread waiting for buffer to become mapped */ -+ wake_up_all(&mem->wait_queue); -+ } else { -+ MALI_DEBUG_ASSERT(mem->is_mapped); -+ mali_session_memory_unlock(session); -+ } -+ -+ return 0; -+} -+ -+static void mali_dma_buf_unmap(mali_mem_allocation *alloc, struct mali_dma_buf_attachment *mem) -+{ -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ MALI_DEBUG_ASSERT_POINTER(mem); -+ MALI_DEBUG_ASSERT_POINTER(mem->attachment); -+ MALI_DEBUG_ASSERT_POINTER(mem->buf); -+ MALI_DEBUG_ASSERT_POINTER(alloc->session); -+ -+ mali_session_memory_lock(alloc->session); -+ mem->map_ref--; -+ -+ MALI_DEBUG_PRINT(5, ("Mali DMA-buf: unmap attachment %p, new map_ref = %d\n", mem, mem->map_ref)); -+ -+ if (0 == mem->map_ref) { -+ dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); -+ if (MALI_TRUE == mem->is_mapped) { -+ mali_mem_mali_map_free(alloc->session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ } -+ mem->is_mapped = MALI_FALSE; -+ } -+ mali_session_memory_unlock(alloc->session); -+ /* Wake up any thread waiting for buffer to become unmapped */ -+ wake_up_all(&mem->wait_queue); -+} -+ -+#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -+int mali_dma_buf_map_job(struct mali_pp_job *job) -+{ -+ struct mali_dma_buf_attachment *mem; -+ _mali_osk_errcode_t err; -+ int i; -+ int ret = 0; -+ u32 num_memory_cookies; -+ struct mali_session_data *session; -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_allocation *mali_alloc = NULL; -+ mali_mem_backend *mem_bkend = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(job); -+ -+ num_memory_cookies = mali_pp_job_num_memory_cookies(job); -+ -+ session = mali_pp_job_get_session(job); -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ for (i = 0; i < num_memory_cookies; i++) { -+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); -+ MALI_DEBUG_ASSERT(NULL != mali_vma_node); -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ MALI_DEBUG_ASSERT(NULL != mali_alloc); -+ if (MALI_MEM_DMA_BUF != mali_alloc->type) { -+ continue; -+ } -+ -+ /* Get backend memory & Map on CPU */ -+ mutex_lock(&mali_idr_mutex); -+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(NULL != mem_bkend); -+ -+ mem = mem_bkend->dma_buf.attachment; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem); -+ MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); -+ -+ err = mali_dma_buf_map(mem_bkend); -+ if (0 != err) { -+ MALI_DEBUG_PRINT_ERROR(("Mali DMA-buf: Failed to map dma-buf for mali address %x\n", mali_addr)); -+ ret = -EFAULT; -+ continue; -+ } -+ } -+ return ret; -+} -+ -+void mali_dma_buf_unmap_job(struct mali_pp_job *job) -+{ -+ struct mali_dma_buf_attachment *mem; -+ int i; -+ u32 num_memory_cookies; -+ struct mali_session_data *session; -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_allocation *mali_alloc = NULL; -+ mali_mem_backend *mem_bkend = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(job); -+ -+ num_memory_cookies = mali_pp_job_num_memory_cookies(job); -+ -+ session = mali_pp_job_get_session(job); -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ for (i = 0; i < num_memory_cookies; i++) { -+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); -+ MALI_DEBUG_ASSERT(NULL != mali_vma_node); -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ MALI_DEBUG_ASSERT(NULL != mali_alloc); -+ if (MALI_MEM_DMA_BUF != mali_alloc->type) { -+ continue; -+ } -+ -+ /* Get backend memory & Map on CPU */ -+ mutex_lock(&mali_idr_mutex); -+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(NULL != mem_bkend); -+ -+ mem = mem_bkend->dma_buf.attachment; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem); -+ MALI_DEBUG_ASSERT(mem->session == mali_pp_job_get_session(job)); -+ mali_dma_buf_unmap(mem_bkend->mali_allocation, mem); -+ } -+} -+#endif /* !CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH */ -+ -+int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *user_arg) -+{ -+ _mali_uk_dma_buf_get_size_s args; -+ int fd; -+ struct dma_buf *buf; -+ -+ /* get call arguments from user space. copy_from_user returns how many bytes which where NOT copied */ -+ if (0 != copy_from_user(&args, (void __user *)user_arg, sizeof(_mali_uk_dma_buf_get_size_s))) { -+ return -EFAULT; -+ } -+ -+ /* Do DMA-BUF stuff */ -+ fd = args.mem_fd; -+ -+ buf = dma_buf_get(fd); -+ if (IS_ERR_OR_NULL(buf)) { -+ MALI_DEBUG_PRINT_ERROR(("Failed to get dma-buf from fd: %d\n", fd)); -+ return PTR_ERR_OR_ZERO(buf); -+ } -+ -+ if (0 != put_user(buf->size, &user_arg->size)) { -+ dma_buf_put(buf); -+ return -EFAULT; -+ } -+ -+ dma_buf_put(buf); -+ -+ return 0; -+} -+ -+_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, -+ mali_mem_backend *mem_backend, -+ int fd, u32 flags) -+{ -+ struct dma_buf *buf; -+ struct mali_dma_buf_attachment *dma_mem; -+ struct mali_session_data *session = alloc->session; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ /* get dma buffer */ -+ buf = dma_buf_get(fd); -+ if (IS_ERR_OR_NULL(buf)) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ /* Currently, mapping of the full buffer are supported. */ -+ if (alloc->psize != buf->size) { -+ goto failed_alloc_mem; -+ } -+ -+ dma_mem = _mali_osk_calloc(1, sizeof(struct mali_dma_buf_attachment)); -+ if (NULL == dma_mem) { -+ goto failed_alloc_mem; -+ } -+ -+ dma_mem->buf = buf; -+ dma_mem->session = session; -+ dma_mem->map_ref = 0; -+ init_waitqueue_head(&dma_mem->wait_queue); -+ -+ dma_mem->attachment = dma_buf_attach(dma_mem->buf, &mali_platform_device->dev); -+ if (NULL == dma_mem->attachment) { -+ goto failed_dma_attach; -+ } -+ -+ mem_backend->dma_buf.attachment = dma_mem; -+ -+ alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; -+ if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { -+ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; -+ } -+ -+ -+#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -+ /* Map memory into session's Mali virtual address space. */ -+ if (0 != mali_dma_buf_map(mem_backend)) { -+ goto Failed_dma_map; -+ } -+#endif -+ -+ return _MALI_OSK_ERR_OK; -+ -+#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -+Failed_dma_map: -+ mali_dma_buf_unmap(alloc, dma_mem); -+#endif -+ /* Wait for buffer to become unmapped */ -+ wait_event(dma_mem->wait_queue, !dma_mem->is_mapped); -+ MALI_DEBUG_ASSERT(!dma_mem->is_mapped); -+ dma_buf_detach(dma_mem->buf, dma_mem->attachment); -+failed_dma_attach: -+ _mali_osk_free(dma_mem); -+failed_alloc_mem: -+ dma_buf_put(buf); -+ return _MALI_OSK_ERR_FAULT; -+} -+ -+void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend) -+{ -+ struct mali_dma_buf_attachment *mem; -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT(MALI_MEM_DMA_BUF == mem_backend->type); -+ -+ mem = mem_backend->dma_buf.attachment; -+ MALI_DEBUG_ASSERT_POINTER(mem); -+ MALI_DEBUG_ASSERT_POINTER(mem->attachment); -+ MALI_DEBUG_ASSERT_POINTER(mem->buf); -+ MALI_DEBUG_PRINT(3, ("Mali DMA-buf: release attachment %p\n", mem)); -+ -+#if defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -+ MALI_DEBUG_ASSERT_POINTER(mem_backend->mali_allocation); -+ /* We mapped implicitly on attach, so we need to unmap on release */ -+ mali_dma_buf_unmap(mem_backend->mali_allocation, mem); -+#endif -+ /* Wait for buffer to become unmapped */ -+ wait_event(mem->wait_queue, !mem->is_mapped); -+ MALI_DEBUG_ASSERT(!mem->is_mapped); -+ -+ dma_buf_detach(mem->buf, mem->attachment); -+ dma_buf_put(mem->buf); -+ -+ _mali_osk_free(mem); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h -new file mode 100755 -index 000000000..a9b287038 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_dma_buf.h -@@ -0,0 +1,53 @@ -+/* -+ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_DMA_BUF_H__ -+#define __MALI_MEMORY_DMA_BUF_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include "mali_uk_types.h" -+#include "mali_osk.h" -+#include "mali_memory.h" -+ -+struct mali_pp_job; -+ -+struct mali_dma_buf_attachment; -+struct mali_dma_buf_attachment { -+ struct dma_buf *buf; -+ struct dma_buf_attachment *attachment; -+ struct sg_table *sgt; -+ struct mali_session_data *session; -+ int map_ref; -+ struct mutex map_lock; -+ mali_bool is_mapped; -+ wait_queue_head_t wait_queue; -+}; -+ -+int mali_dma_buf_get_size(struct mali_session_data *session, _mali_uk_dma_buf_get_size_s __user *arg); -+ -+void mali_mem_unbind_dma_buf(mali_mem_backend *mem_backend); -+ -+_mali_osk_errcode_t mali_mem_bind_dma_buf(mali_mem_allocation *alloc, -+ mali_mem_backend *mem_backend, -+ int fd, u32 flags); -+ -+#if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) -+int mali_dma_buf_map_job(struct mali_pp_job *job); -+void mali_dma_buf_unmap_job(struct mali_pp_job *job); -+#endif -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_MEMORY_DMA_BUF_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c -new file mode 100755 -index 000000000..76018b7ab ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.c -@@ -0,0 +1,89 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_ukk.h" -+#include "mali_memory.h" -+#include "mali_mem_validation.h" -+#include "mali_uk_types.h" -+ -+void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend) -+{ -+ mali_mem_allocation *alloc; -+ struct mali_session_data *session; -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ alloc = mem_backend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ MALI_DEBUG_ASSERT(MALI_MEM_EXTERNAL == mem_backend->type); -+ -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ mali_session_memory_lock(session); -+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ mali_session_memory_unlock(session); -+} -+ -+_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, -+ mali_mem_backend *mem_backend, -+ u32 phys_addr, -+ u32 flag) -+{ -+ struct mali_session_data *session; -+ _mali_osk_errcode_t err; -+ u32 virt, phys, size; -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ size = alloc->psize; -+ session = (struct mali_session_data *)(uintptr_t)alloc->session; -+ MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_INVALID_ARGS); -+ -+ /* check arguments */ -+ /* NULL might be a valid Mali address */ -+ if (!size) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ -+ /* size must be a multiple of the system page size */ -+ if (size % _MALI_OSK_MALI_PAGE_SIZE) MALI_ERROR(_MALI_OSK_ERR_INVALID_ARGS); -+ -+ /* Validate the mali physical range */ -+ if (_MALI_OSK_ERR_OK != mali_mem_validation_check(phys_addr, size)) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ if (flag & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { -+ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; -+ } -+ -+ mali_session_memory_lock(session); -+ -+ virt = alloc->mali_vma_node.vm_node.start; -+ phys = phys_addr; -+ -+ err = mali_mem_mali_map_prepare(alloc); -+ if (_MALI_OSK_ERR_OK != err) { -+ mali_session_memory_unlock(session); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ mali_mmu_pagedir_update(session->page_directory, virt, phys, size, MALI_MMU_FLAGS_DEFAULT); -+ -+ if (alloc->flags & MALI_MEM_FLAG_MALI_GUARD_PAGE) { -+ mali_mmu_pagedir_update(session->page_directory, virt + size, phys, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); -+ } -+ MALI_DEBUG_PRINT(3, -+ ("Requested to map physical memory 0x%x-0x%x into virtual memory 0x%x\n", -+ phys_addr, (phys_addr + size - 1), -+ virt)); -+ mali_session_memory_unlock(session); -+ -+ MALI_SUCCESS; -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h -new file mode 100755 -index 000000000..2db178d96 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_external.h -@@ -0,0 +1,29 @@ -+ -+/* -+ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_EXTERNAL_H__ -+#define __MALI_MEMORY_EXTERNAL_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+_mali_osk_errcode_t mali_mem_bind_ext_buf(mali_mem_allocation *alloc, -+ mali_mem_backend *mem_backend, -+ u32 phys_addr, -+ u32 flag); -+void mali_mem_unbind_ext_buf(mali_mem_backend *mem_backend); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c -new file mode 100755 -index 000000000..27dee0f19 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.c -@@ -0,0 +1,993 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+#include -+#endif -+#include -+ -+#include "mali_osk.h" -+#include "mali_osk_mali.h" -+#include "mali_kernel_linux.h" -+#include "mali_scheduler.h" -+#include "mali_memory.h" -+#include "mali_memory_os_alloc.h" -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+#include "mali_memory_dma_buf.h" -+#include "mali_memory_secure.h" -+#endif -+#if defined(CONFIG_MALI400_UMP) -+#include "mali_memory_ump.h" -+#endif -+#include "mali_memory_manager.h" -+#include "mali_memory_virtual.h" -+#include "mali_memory_util.h" -+#include "mali_memory_external.h" -+#include "mali_memory_cow.h" -+#include "mali_memory_block_alloc.h" -+#include "mali_ukk.h" -+#include "mali_memory_swap_alloc.h" -+ -+/* -+* New memory system interface -+*/ -+ -+/*inti idr for backend memory */ -+struct idr mali_backend_idr; -+struct mutex mali_idr_mutex; -+ -+/* init allocation manager */ -+int mali_memory_manager_init(struct mali_allocation_manager *mgr) -+{ -+ /* init Locks */ -+ rwlock_init(&mgr->vm_lock); -+ mutex_init(&mgr->list_mutex); -+ -+ /* init link */ -+ INIT_LIST_HEAD(&mgr->head); -+ -+ /* init RB tree */ -+ mgr->allocation_mgr_rb = RB_ROOT; -+ mgr->mali_allocation_num = 0; -+ return 0; -+} -+ -+/* Deinit allocation manager -+* Do some check for debug -+*/ -+void mali_memory_manager_uninit(struct mali_allocation_manager *mgr) -+{ -+ /* check RB tree is empty */ -+ MALI_DEBUG_ASSERT(((void *)(mgr->allocation_mgr_rb.rb_node) == (void *)rb_last(&mgr->allocation_mgr_rb))); -+ /* check allocation List */ -+ MALI_DEBUG_ASSERT(list_empty(&mgr->head)); -+} -+ -+/* Prepare memory descriptor */ -+static mali_mem_allocation *mali_mem_allocation_struct_create(struct mali_session_data *session) -+{ -+ mali_mem_allocation *mali_allocation; -+ -+ /* Allocate memory */ -+ mali_allocation = (mali_mem_allocation *)kzalloc(sizeof(mali_mem_allocation), GFP_KERNEL); -+ if (NULL == mali_allocation) { -+ MALI_DEBUG_PRINT(1, ("mali_mem_allocation_struct_create: descriptor was NULL\n")); -+ return NULL; -+ } -+ -+ MALI_DEBUG_CODE(mali_allocation->magic = MALI_MEM_ALLOCATION_VALID_MAGIC); -+ -+ /* do init */ -+ mali_allocation->flags = 0; -+ mali_allocation->session = session; -+ -+ INIT_LIST_HEAD(&mali_allocation->list); -+ _mali_osk_atomic_init(&mali_allocation->mem_alloc_refcount, 1); -+ -+ /** -+ *add to session list -+ */ -+ mutex_lock(&session->allocation_mgr.list_mutex); -+ list_add_tail(&mali_allocation->list, &session->allocation_mgr.head); -+ session->allocation_mgr.mali_allocation_num++; -+ mutex_unlock(&session->allocation_mgr.list_mutex); -+ -+ return mali_allocation; -+} -+ -+void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc) -+{ -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ MALI_DEBUG_ASSERT_POINTER(alloc->session); -+ mutex_lock(&alloc->session->allocation_mgr.list_mutex); -+ list_del(&alloc->list); -+ alloc->session->allocation_mgr.mali_allocation_num--; -+ mutex_unlock(&alloc->session->allocation_mgr.list_mutex); -+ -+ kfree(alloc); -+} -+ -+int mali_mem_backend_struct_create(mali_mem_backend **backend, u32 psize) -+{ -+ mali_mem_backend *mem_backend = NULL; -+ s32 ret = -ENOSPC; -+ s32 index = -1; -+ *backend = (mali_mem_backend *)kzalloc(sizeof(mali_mem_backend), GFP_KERNEL); -+ if (NULL == *backend) { -+ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: backend descriptor was NULL\n")); -+ return -1; -+ } -+ mem_backend = *backend; -+ mem_backend->size = psize; -+ mutex_init(&mem_backend->mutex); -+ INIT_LIST_HEAD(&mem_backend->list); -+ mem_backend->using_count = 0; -+ -+ -+ /* link backend with id */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) -+again: -+ if (!idr_pre_get(&mali_backend_idr, GFP_KERNEL)) { -+ kfree(mem_backend); -+ return -ENOMEM; -+ } -+ mutex_lock(&mali_idr_mutex); -+ ret = idr_get_new_above(&mali_backend_idr, mem_backend, 1, &index); -+ mutex_unlock(&mali_idr_mutex); -+ -+ if (-ENOSPC == ret) { -+ kfree(mem_backend); -+ return -ENOSPC; -+ } -+ if (-EAGAIN == ret) -+ goto again; -+#else -+ mutex_lock(&mali_idr_mutex); -+ ret = idr_alloc(&mali_backend_idr, mem_backend, 1, MALI_S32_MAX, GFP_KERNEL); -+ mutex_unlock(&mali_idr_mutex); -+ index = ret; -+ if (ret < 0) { -+ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_create: Can't allocate idr for backend! \n")); -+ kfree(mem_backend); -+ return -ENOSPC; -+ } -+#endif -+ return index; -+} -+ -+ -+static void mali_mem_backend_struct_destory(mali_mem_backend **backend, s32 backend_handle) -+{ -+ mali_mem_backend *mem_backend = *backend; -+ -+ mutex_lock(&mali_idr_mutex); -+ idr_remove(&mali_backend_idr, backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ kfree(mem_backend); -+ *backend = NULL; -+} -+ -+mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address) -+{ -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_backend *mem_bkend = NULL; -+ mali_mem_allocation *mali_alloc = NULL; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_address, 0); -+ if (NULL == mali_vma_node) { -+ MALI_DEBUG_PRINT(1, ("mali_mem_backend_struct_search:vma node was NULL\n")); -+ return NULL; -+ } -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ /* Get backend memory & Map on CPU */ -+ mutex_lock(&mali_idr_mutex); -+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(NULL != mem_bkend); -+ return mem_bkend; -+} -+ -+static _mali_osk_errcode_t mali_mem_resize(struct mali_session_data *session, mali_mem_backend *mem_backend, u32 physical_size) -+{ -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ int retval = 0; -+ mali_mem_allocation *mali_allocation = NULL; -+ mali_mem_os_mem tmp_os_mem; -+ s32 change_page_count; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); -+ MALI_DEBUG_ASSERT(0 == physical_size % MALI_MMU_PAGE_SIZE); -+ -+ mali_allocation = mem_backend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(mali_allocation); -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE & mali_allocation->flags); -+ MALI_DEBUG_ASSERT(MALI_MEM_OS == mali_allocation->type); -+ -+ mutex_lock(&mem_backend->mutex); -+ -+ /* Do resize*/ -+ if (physical_size > mem_backend->size) { -+ u32 add_size = physical_size - mem_backend->size; -+ -+ MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); -+ -+ /* Allocate new pages from os mem */ -+ retval = mali_mem_os_alloc_pages(&tmp_os_mem, add_size); -+ -+ if (retval) { -+ if (-ENOMEM == retval) { -+ ret = _MALI_OSK_ERR_NOMEM; -+ } else { -+ ret = _MALI_OSK_ERR_FAULT; -+ } -+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory allocation failed !\n")); -+ goto failed_alloc_memory; -+ } -+ -+ MALI_DEBUG_ASSERT(tmp_os_mem.count == add_size / MALI_MMU_PAGE_SIZE); -+ -+ /* Resize the memory of the backend */ -+ ret = mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); -+ -+ if (ret) { -+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory resizing failed !\n")); -+ goto failed_resize_pages; -+ } -+ -+ /*Resize cpu mapping */ -+ if (NULL != mali_allocation->cpu_mapping.vma) { -+ ret = mali_mem_os_resize_cpu_map_locked(mem_backend, mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + mem_backend->size, add_size); -+ if (unlikely(ret != _MALI_OSK_ERR_OK)) { -+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: cpu mapping failed !\n")); -+ goto failed_cpu_map; -+ } -+ } -+ -+ /* Resize mali mapping */ -+ _mali_osk_mutex_wait(session->memory_lock); -+ ret = mali_mem_mali_map_resize(mali_allocation, physical_size); -+ -+ if (ret) { -+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_resize: mali map resize fail !\n")); -+ goto failed_gpu_map; -+ } -+ -+ ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, mali_allocation->mali_vma_node.vm_node.start, -+ mali_allocation->psize / MALI_MMU_PAGE_SIZE, add_size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); -+ if (ret) { -+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: mali mapping failed !\n")); -+ goto failed_gpu_map; -+ } -+ -+ _mali_osk_mutex_signal(session->memory_lock); -+ } else { -+ u32 dec_size, page_count; -+ u32 vaddr = 0; -+ INIT_LIST_HEAD(&tmp_os_mem.pages); -+ tmp_os_mem.count = 0; -+ -+ dec_size = mem_backend->size - physical_size; -+ MALI_DEBUG_ASSERT(0 == dec_size % MALI_MMU_PAGE_SIZE); -+ -+ page_count = dec_size / MALI_MMU_PAGE_SIZE; -+ vaddr = mali_allocation->mali_vma_node.vm_node.start + physical_size; -+ -+ /* Resize the memory of the backend */ -+ ret = mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, physical_size / MALI_MMU_PAGE_SIZE, page_count); -+ -+ if (ret) { -+ MALI_DEBUG_PRINT(4, ("_mali_ukk_mem_resize: mali map resize failed!\n")); -+ goto failed_resize_pages; -+ } -+ -+ /* Resize mali map */ -+ _mali_osk_mutex_wait(session->memory_lock); -+ mali_mem_mali_map_free(session, dec_size, vaddr, mali_allocation->flags); -+ _mali_osk_mutex_signal(session->memory_lock); -+ -+ /* Zap cpu mapping */ -+ if (0 != mali_allocation->cpu_mapping.addr) { -+ MALI_DEBUG_ASSERT(NULL != mali_allocation->cpu_mapping.vma); -+ zap_vma_ptes(mali_allocation->cpu_mapping.vma, mali_allocation->cpu_mapping.vma->vm_start + physical_size, dec_size); -+ } -+ -+ /* Free those extra pages */ -+ mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); -+ } -+ -+ /* Resize memory allocation and memory backend */ -+ change_page_count = (s32)(physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE; -+ mali_allocation->psize = physical_size; -+ mem_backend->size = physical_size; -+ mutex_unlock(&mem_backend->mutex); -+ -+ if (change_page_count > 0) { -+ atomic_add(change_page_count, &session->mali_mem_allocated_pages); -+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { -+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ -+ } else { -+ atomic_sub((s32)(-change_page_count), &session->mali_mem_allocated_pages); -+ } -+ -+ return _MALI_OSK_ERR_OK; -+ -+failed_gpu_map: -+ _mali_osk_mutex_signal(session->memory_lock); -+failed_cpu_map: -+ if (physical_size > mem_backend->size) { -+ mali_mem_os_resize_pages(&mem_backend->os_mem, &tmp_os_mem, mem_backend->size / MALI_MMU_PAGE_SIZE, -+ (physical_size - mem_backend->size) / MALI_MMU_PAGE_SIZE); -+ } else { -+ mali_mem_os_resize_pages(&tmp_os_mem, &mem_backend->os_mem, 0, tmp_os_mem.count); -+ } -+failed_resize_pages: -+ if (0 != tmp_os_mem.count) -+ mali_mem_os_free(&tmp_os_mem.pages, tmp_os_mem.count, MALI_FALSE); -+failed_alloc_memory: -+ -+ mutex_unlock(&mem_backend->mutex); -+ return ret; -+} -+ -+ -+/* Set GPU MMU properties */ -+static void _mali_memory_gpu_map_property_set(u32 *properties, u32 flags) -+{ -+ if (_MALI_MEMORY_GPU_READ_ALLOCATE & flags) { -+ *properties = MALI_MMU_FLAGS_FORCE_GP_READ_ALLOCATE; -+ } else { -+ *properties = MALI_MMU_FLAGS_DEFAULT; -+ } -+} -+ -+_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size) -+{ -+ mali_mem_backend *mem_backend = NULL; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ mali_mem_allocation *mali_allocation = NULL; -+ u32 new_physical_size; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_ASSERT(0 == add_size % MALI_MMU_PAGE_SIZE); -+ -+ /* Get the memory backend that need to be resize. */ -+ mem_backend = mali_mem_backend_struct_search(session, mali_addr); -+ -+ if (NULL == mem_backend) { -+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); -+ return ret; -+ } -+ -+ mali_allocation = mem_backend->mali_allocation; -+ -+ MALI_DEBUG_ASSERT_POINTER(mali_allocation); -+ -+ new_physical_size = add_size + mem_backend->size; -+ -+ if (new_physical_size > (mali_allocation->mali_vma_node.vm_node.size)) -+ return ret; -+ -+ MALI_DEBUG_ASSERT(new_physical_size != mem_backend->size); -+ -+ ret = mali_mem_resize(session, mem_backend, new_physical_size); -+ -+ return ret; -+} -+ -+/** -+* function@_mali_ukk_mem_allocate - allocate mali memory -+*/ -+_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args) -+{ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ mali_mem_backend *mem_backend = NULL; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ int retval = 0; -+ mali_mem_allocation *mali_allocation = NULL; -+ struct mali_vma_node *mali_vma_node = NULL; -+ -+ MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_allocate, vaddr=0x%x, size =0x%x! \n", args->gpu_vaddr, args->psize)); -+ -+ /* Check if the address is allocated -+ */ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->gpu_vaddr, 0); -+ -+ if (unlikely(mali_vma_node)) { -+ MALI_DEBUG_PRINT_ERROR(("The mali virtual address has already been used ! \n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ /** -+ *create mali memory allocation -+ */ -+ -+ mali_allocation = mali_mem_allocation_struct_create(session); -+ -+ if (mali_allocation == NULL) { -+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_allocate: Failed to create allocation struct! \n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ mali_allocation->psize = args->psize; -+ mali_allocation->vsize = args->vsize; -+ -+ /* MALI_MEM_OS if need to support mem resize, -+ * or MALI_MEM_BLOCK if have dedicated memory, -+ * or MALI_MEM_OS, -+ * or MALI_MEM_SWAP. -+ */ -+ if (args->flags & _MALI_MEMORY_ALLOCATE_SWAPPABLE) { -+ mali_allocation->type = MALI_MEM_SWAP; -+ } else if (args->flags & _MALI_MEMORY_ALLOCATE_RESIZEABLE) { -+ mali_allocation->type = MALI_MEM_OS; -+ mali_allocation->flags |= MALI_MEM_FLAG_CAN_RESIZE; -+ } else if (args->flags & _MALI_MEMORY_ALLOCATE_SECURE) { -+ mali_allocation->type = MALI_MEM_SECURE; -+ } else if (MALI_TRUE == mali_memory_have_dedicated_memory()) { -+ mali_allocation->type = MALI_MEM_BLOCK; -+ } else { -+ mali_allocation->type = MALI_MEM_OS; -+ } -+ -+ /** -+ *add allocation node to RB tree for index -+ */ -+ mali_allocation->mali_vma_node.vm_node.start = args->gpu_vaddr; -+ mali_allocation->mali_vma_node.vm_node.size = args->vsize; -+ -+ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); -+ -+ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, args->psize); -+ if (mali_allocation->backend_handle < 0) { -+ ret = _MALI_OSK_ERR_NOMEM; -+ MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); -+ goto failed_alloc_backend; -+ } -+ -+ -+ mem_backend->mali_allocation = mali_allocation; -+ mem_backend->type = mali_allocation->type; -+ -+ mali_allocation->mali_mapping.addr = args->gpu_vaddr; -+ -+ /* set gpu mmu propery */ -+ _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); -+ /* do prepare for MALI mapping */ -+ if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { -+ _mali_osk_mutex_wait(session->memory_lock); -+ -+ ret = mali_mem_mali_map_prepare(mali_allocation); -+ if (0 != ret) { -+ _mali_osk_mutex_signal(session->memory_lock); -+ goto failed_prepare_map; -+ } -+ _mali_osk_mutex_signal(session->memory_lock); -+ } -+ -+ if (mali_allocation->psize == 0) { -+ mem_backend->os_mem.count = 0; -+ INIT_LIST_HEAD(&mem_backend->os_mem.pages); -+ goto done; -+ } -+ -+ if (args->flags & _MALI_MEMORY_ALLOCATE_DEFER_BIND) { -+ mali_allocation->flags |= _MALI_MEMORY_ALLOCATE_DEFER_BIND; -+ mem_backend->flags |= MALI_MEM_BACKEND_FLAG_NOT_BINDED; -+ /* init for defer bind backend*/ -+ mem_backend->os_mem.count = 0; -+ INIT_LIST_HEAD(&mem_backend->os_mem.pages); -+ -+ goto done; -+ } -+ -+ if (likely(mali_allocation->psize > 0)) { -+ -+ if (MALI_MEM_SECURE == mem_backend->type) { -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ ret = mali_mem_secure_attach_dma_buf(&mem_backend->secure_mem, mem_backend->size, args->secure_shared_fd); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("Failed to attach dma buf for secure memory! \n")); -+ goto failed_alloc_pages; -+ } -+#else -+ ret = _MALI_OSK_ERR_UNSUPPORTED; -+ MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory! \n")); -+ goto failed_alloc_pages; -+#endif -+ } else { -+ -+ /** -+ *allocate physical memory -+ */ -+ if (mem_backend->type == MALI_MEM_OS) { -+ retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); -+ } else if (mem_backend->type == MALI_MEM_BLOCK) { -+ /* try to allocated from BLOCK memory first, then try OS memory if failed.*/ -+ if (mali_mem_block_alloc(&mem_backend->block_mem, mem_backend->size)) { -+ retval = mali_mem_os_alloc_pages(&mem_backend->os_mem, mem_backend->size); -+ mem_backend->type = MALI_MEM_OS; -+ mali_allocation->type = MALI_MEM_OS; -+ } -+ } else if (MALI_MEM_SWAP == mem_backend->type) { -+ retval = mali_mem_swap_alloc_pages(&mem_backend->swap_mem, mali_allocation->mali_vma_node.vm_node.size, &mem_backend->start_idx); -+ } else { -+ /* ONLY support mem_os type */ -+ MALI_DEBUG_ASSERT(0); -+ } -+ -+ if (retval) { -+ ret = _MALI_OSK_ERR_NOMEM; -+ MALI_DEBUG_PRINT(1, (" can't allocate enough pages! \n")); -+ goto failed_alloc_pages; -+ } -+ } -+ } -+ -+ /** -+ *map to GPU side -+ */ -+ if (!(args->flags & _MALI_MEMORY_ALLOCATE_NO_BIND_GPU) && mali_allocation->psize > 0) { -+ _mali_osk_mutex_wait(session->memory_lock); -+ /* Map on Mali */ -+ -+ if (mem_backend->type == MALI_MEM_OS) { -+ ret = mali_mem_os_mali_map(&mem_backend->os_mem, session, args->gpu_vaddr, 0, -+ mem_backend->size / MALI_MMU_PAGE_SIZE, mali_allocation->mali_mapping.properties); -+ -+ } else if (mem_backend->type == MALI_MEM_BLOCK) { -+ mali_mem_block_mali_map(&mem_backend->block_mem, session, args->gpu_vaddr, -+ mali_allocation->mali_mapping.properties); -+ } else if (mem_backend->type == MALI_MEM_SWAP) { -+ ret = mali_mem_swap_mali_map(&mem_backend->swap_mem, session, args->gpu_vaddr, -+ mali_allocation->mali_mapping.properties); -+ } else if (mem_backend->type == MALI_MEM_SECURE) { -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ ret = mali_mem_secure_mali_map(&mem_backend->secure_mem, session, args->gpu_vaddr, mali_allocation->mali_mapping.properties); -+#endif -+ } else { /* unsupport type */ -+ MALI_DEBUG_ASSERT(0); -+ } -+ -+ _mali_osk_mutex_signal(session->memory_lock); -+ } -+done: -+ if (MALI_MEM_OS == mem_backend->type) { -+ atomic_add(mem_backend->os_mem.count, &session->mali_mem_allocated_pages); -+ } else if (MALI_MEM_BLOCK == mem_backend->type) { -+ atomic_add(mem_backend->block_mem.count, &session->mali_mem_allocated_pages); -+ } else if (MALI_MEM_SECURE == mem_backend->type) { -+ atomic_add(mem_backend->secure_mem.count, &session->mali_mem_allocated_pages); -+ } else { -+ MALI_DEBUG_ASSERT(MALI_MEM_SWAP == mem_backend->type); -+ atomic_add(mem_backend->swap_mem.count, &session->mali_mem_allocated_pages); -+ atomic_add(mem_backend->swap_mem.count, &session->mali_mem_array[mem_backend->type]); -+ } -+ -+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { -+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ return _MALI_OSK_ERR_OK; -+ -+failed_alloc_pages: -+ mali_mem_mali_map_free(session, mali_allocation->psize, mali_allocation->mali_vma_node.vm_node.start, mali_allocation->flags); -+failed_prepare_map: -+ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); -+failed_alloc_backend: -+ -+ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); -+ mali_mem_allocation_struct_destory(mali_allocation); -+ -+ return ret; -+} -+ -+ -+_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args) -+{ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ u32 vaddr = args->gpu_vaddr; -+ mali_mem_allocation *mali_alloc = NULL; -+ struct mali_vma_node *mali_vma_node = NULL; -+ -+ /* find mali allocation structure by vaddress*/ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, vaddr, 0); -+ if (NULL == mali_vma_node) { -+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_free: invalid addr: 0x%x\n", vaddr)); -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+ MALI_DEBUG_ASSERT(NULL != mali_vma_node); -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ -+ if (mali_alloc) -+ /* check ref_count */ -+ args->free_pages_nr = mali_allocation_unref(&mali_alloc); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+/** -+* Function _mali_ukk_mem_bind -- bind a external memory to a new GPU address -+* It will allocate a new mem allocation and bind external memory to it. -+* Supported backend type are: -+* _MALI_MEMORY_BIND_BACKEND_UMP -+* _MALI_MEMORY_BIND_BACKEND_DMA_BUF -+* _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY -+* CPU access is not supported yet -+*/ -+_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args) -+{ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ mali_mem_backend *mem_backend = NULL; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ mali_mem_allocation *mali_allocation = NULL; -+ MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_bind, vaddr=0x%x, size =0x%x! \n", args->vaddr, args->size)); -+ -+ /** -+ * allocate mali allocation. -+ */ -+ mali_allocation = mali_mem_allocation_struct_create(session); -+ -+ if (mali_allocation == NULL) { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ mali_allocation->psize = args->size; -+ mali_allocation->vsize = args->size; -+ mali_allocation->mali_mapping.addr = args->vaddr; -+ -+ /* add allocation node to RB tree for index */ -+ mali_allocation->mali_vma_node.vm_node.start = args->vaddr; -+ mali_allocation->mali_vma_node.vm_node.size = args->size; -+ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); -+ -+ /* allocate backend*/ -+ if (mali_allocation->psize > 0) { -+ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); -+ if (mali_allocation->backend_handle < 0) { -+ goto Failed_alloc_backend; -+ } -+ -+ } else { -+ goto Failed_alloc_backend; -+ } -+ -+ mem_backend->size = mali_allocation->psize; -+ mem_backend->mali_allocation = mali_allocation; -+ -+ switch (args->flags & _MALI_MEMORY_BIND_BACKEND_MASK) { -+ case _MALI_MEMORY_BIND_BACKEND_UMP: -+#if defined(CONFIG_MALI400_UMP) -+ mali_allocation->type = MALI_MEM_UMP; -+ mem_backend->type = MALI_MEM_UMP; -+ ret = mali_mem_bind_ump_buf(mali_allocation, mem_backend, -+ args->mem_union.bind_ump.secure_id, args->mem_union.bind_ump.flags); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("Bind ump buf failed\n")); -+ goto Failed_bind_backend; -+ } -+#else -+ MALI_DEBUG_PRINT(1, ("UMP not supported\n")); -+ goto Failed_bind_backend; -+#endif -+ break; -+ case _MALI_MEMORY_BIND_BACKEND_DMA_BUF: -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ mali_allocation->type = MALI_MEM_DMA_BUF; -+ mem_backend->type = MALI_MEM_DMA_BUF; -+ ret = mali_mem_bind_dma_buf(mali_allocation, mem_backend, -+ args->mem_union.bind_dma_buf.mem_fd, args->mem_union.bind_dma_buf.flags); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("Bind dma buf failed\n")); -+ goto Failed_bind_backend; -+ } -+#else -+ MALI_DEBUG_PRINT(1, ("DMA not supported\n")); -+ goto Failed_bind_backend; -+#endif -+ break; -+ case _MALI_MEMORY_BIND_BACKEND_MALI_MEMORY: -+ /* not allowed */ -+ MALI_DEBUG_PRINT_ERROR(("Mali internal memory type not supported !\n")); -+ goto Failed_bind_backend; -+ break; -+ -+ case _MALI_MEMORY_BIND_BACKEND_EXTERNAL_MEMORY: -+ mali_allocation->type = MALI_MEM_EXTERNAL; -+ mem_backend->type = MALI_MEM_EXTERNAL; -+ ret = mali_mem_bind_ext_buf(mali_allocation, mem_backend, args->mem_union.bind_ext_memory.phys_addr, -+ args->mem_union.bind_ext_memory.flags); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("Bind external buf failed\n")); -+ goto Failed_bind_backend; -+ } -+ break; -+ -+ case _MALI_MEMORY_BIND_BACKEND_EXT_COW: -+ /* not allowed */ -+ MALI_DEBUG_PRINT_ERROR(("External cow memory type not supported !\n")); -+ goto Failed_bind_backend; -+ break; -+ -+ default: -+ MALI_DEBUG_PRINT_ERROR(("Invalid memory type not supported !\n")); -+ goto Failed_bind_backend; -+ break; -+ } -+ MALI_DEBUG_ASSERT(0 == mem_backend->size % MALI_MMU_PAGE_SIZE); -+ atomic_add(mem_backend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_backend->type]); -+ return _MALI_OSK_ERR_OK; -+ -+Failed_bind_backend: -+ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); -+ -+Failed_alloc_backend: -+ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); -+ mali_mem_allocation_struct_destory(mali_allocation); -+ -+ MALI_DEBUG_PRINT(1, (" _mali_ukk_mem_bind, return ERROR! \n")); -+ return ret; -+} -+ -+ -+/* -+* Function _mali_ukk_mem_unbind -- unbind a external memory to a new GPU address -+* This function unbind the backend memory and free the allocation -+* no ref_count for this type of memory -+*/ -+_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args) -+{ -+ /**/ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ mali_mem_allocation *mali_allocation = NULL; -+ struct mali_vma_node *mali_vma_node = NULL; -+ u32 mali_addr = args->vaddr; -+ MALI_DEBUG_PRINT(5, (" _mali_ukk_mem_unbind, vaddr=0x%x! \n", args->vaddr)); -+ -+ /* find the allocation by vaddr */ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); -+ if (likely(mali_vma_node)) { -+ MALI_DEBUG_ASSERT(mali_addr == mali_vma_node->vm_node.start); -+ mali_allocation = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ } else { -+ MALI_DEBUG_ASSERT(NULL != mali_vma_node); -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+ -+ if (NULL != mali_allocation) -+ /* check ref_count */ -+ mali_allocation_unref(&mali_allocation); -+ return _MALI_OSK_ERR_OK; -+} -+ -+/* -+* Function _mali_ukk_mem_cow -- COW for an allocation -+* This function allocate new pages for a range (range, range+size) of allocation -+* And Map it(keep use the not in range pages from target allocation ) to an GPU vaddr -+*/ -+_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args) -+{ -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ mali_mem_backend *target_backend = NULL; -+ mali_mem_backend *mem_backend = NULL; -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_allocation *mali_allocation = NULL; -+ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ /* Get the target backend for cow */ -+ target_backend = mali_mem_backend_struct_search(session, args->target_handle); -+ -+ if (NULL == target_backend || 0 == target_backend->size) { -+ MALI_DEBUG_ASSERT_POINTER(target_backend); -+ MALI_DEBUG_ASSERT(0 != target_backend->size); -+ return ret; -+ } -+ -+ /*Cow not support resized mem */ -+ MALI_DEBUG_ASSERT(MALI_MEM_FLAG_CAN_RESIZE != (MALI_MEM_FLAG_CAN_RESIZE & target_backend->mali_allocation->flags)); -+ -+ /* Check if the new mali address is allocated */ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, args->vaddr, 0); -+ -+ if (unlikely(mali_vma_node)) { -+ MALI_DEBUG_PRINT_ERROR(("The mali virtual address has already been used ! \n")); -+ return ret; -+ } -+ -+ /* create new alloction for COW*/ -+ mali_allocation = mali_mem_allocation_struct_create(session); -+ if (mali_allocation == NULL) { -+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to create allocation struct!\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ mali_allocation->psize = args->target_size; -+ mali_allocation->vsize = args->target_size; -+ mali_allocation->type = MALI_MEM_COW; -+ -+ /*add allocation node to RB tree for index*/ -+ mali_allocation->mali_vma_node.vm_node.start = args->vaddr; -+ mali_allocation->mali_vma_node.vm_node.size = mali_allocation->vsize; -+ mali_vma_offset_add(&session->allocation_mgr, &mali_allocation->mali_vma_node); -+ -+ /* create new backend for COW memory */ -+ mali_allocation->backend_handle = mali_mem_backend_struct_create(&mem_backend, mali_allocation->psize); -+ if (mali_allocation->backend_handle < 0) { -+ ret = _MALI_OSK_ERR_NOMEM; -+ MALI_DEBUG_PRINT(1, ("mali_allocation->backend_handle < 0! \n")); -+ goto failed_alloc_backend; -+ } -+ mem_backend->mali_allocation = mali_allocation; -+ mem_backend->type = mali_allocation->type; -+ -+ if (target_backend->type == MALI_MEM_SWAP || -+ (MALI_MEM_COW == target_backend->type && (MALI_MEM_BACKEND_FLAG_SWAP_COWED & target_backend->flags))) { -+ mem_backend->flags |= MALI_MEM_BACKEND_FLAG_SWAP_COWED; -+ /** -+ * CoWed swap backends couldn't be mapped as non-linear vma, because if one -+ * vma is set with flag VM_NONLINEAR, the vma->vm_private_data will be used by kernel, -+ * while in mali driver, we use this variable to store the pointer of mali_allocation, so there -+ * is a conflict. -+ * To resolve this problem, we have to do some fake things, we reserved about 64MB -+ * space from index 0, there isn't really page's index will be set from 0 to (64MB>>PAGE_SHIFT_NUM), -+ * and all of CoWed swap memory backends' start_idx will be assigned with 0, and these -+ * backends will be mapped as linear and will add to priority tree of global swap file, while -+ * these vmas will never be found by using normal page->index, these pages in those vma -+ * also couldn't be swapped out. -+ */ -+ mem_backend->start_idx = 0; -+ } -+ -+ /* Add the target backend's cow count, also allocate new pages for COW backend from os mem -+ *for a modified range and keep the page which not in the modified range and Add ref to it -+ */ -+ MALI_DEBUG_PRINT(3, ("Cow mapping: target_addr: 0x%x; cow_addr: 0x%x, size: %u\n", target_backend->mali_allocation->mali_vma_node.vm_node.start, -+ mali_allocation->mali_vma_node.vm_node.start, mali_allocation->mali_vma_node.vm_node.size)); -+ -+ ret = mali_memory_do_cow(target_backend, args->target_offset, args->target_size, mem_backend, args->range_start, args->range_size); -+ if (_MALI_OSK_ERR_OK != ret) { -+ MALI_DEBUG_PRINT(1, ("_mali_ukk_mem_cow: Failed to cow!\n")); -+ goto failed_do_cow; -+ } -+ -+ /** -+ *map to GPU side -+ */ -+ mali_allocation->mali_mapping.addr = args->vaddr; -+ /* set gpu mmu propery */ -+ _mali_memory_gpu_map_property_set(&mali_allocation->mali_mapping.properties, args->flags); -+ -+ _mali_osk_mutex_wait(session->memory_lock); -+ /* Map on Mali */ -+ ret = mali_mem_mali_map_prepare(mali_allocation); -+ if (0 != ret) { -+ MALI_DEBUG_PRINT(1, (" prepare map fail! \n")); -+ goto failed_gpu_map; -+ } -+ -+ if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { -+ mali_mem_cow_mali_map(mem_backend, 0, mem_backend->size); -+ } -+ -+ _mali_osk_mutex_signal(session->memory_lock); -+ -+ mutex_lock(&target_backend->mutex); -+ target_backend->flags |= MALI_MEM_BACKEND_FLAG_COWED; -+ mutex_unlock(&target_backend->mutex); -+ -+ atomic_add(args->range_size / MALI_MMU_PAGE_SIZE, &session->mali_mem_allocated_pages); -+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { -+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ return _MALI_OSK_ERR_OK; -+ -+failed_gpu_map: -+ _mali_osk_mutex_signal(session->memory_lock); -+ mali_mem_cow_release(mem_backend, MALI_FALSE); -+ mem_backend->cow_mem.count = 0; -+failed_do_cow: -+ mali_mem_backend_struct_destory(&mem_backend, mali_allocation->backend_handle); -+failed_alloc_backend: -+ mali_vma_offset_remove(&session->allocation_mgr, &mali_allocation->mali_vma_node); -+ mali_mem_allocation_struct_destory(mali_allocation); -+ -+ return ret; -+} -+ -+_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args) -+{ -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ mali_mem_backend *mem_backend = NULL; -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ -+ MALI_DEBUG_PRINT(4, (" _mali_ukk_mem_cow_modify_range called! \n")); -+ /* Get the backend that need to be modified. */ -+ mem_backend = mali_mem_backend_struct_search(session, args->vaddr); -+ -+ if (NULL == mem_backend || 0 == mem_backend->size) { -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT(0 != mem_backend->size); -+ return ret; -+ } -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_backend->type); -+ -+ ret = mali_memory_cow_modify_range(mem_backend, args->range_start, args->size); -+ args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; -+ if (_MALI_OSK_ERR_OK != ret) -+ return ret; -+ _mali_osk_mutex_wait(session->memory_lock); -+ if (!(mem_backend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)) { -+ mali_mem_cow_mali_map(mem_backend, args->range_start, args->size); -+ } -+ _mali_osk_mutex_signal(session->memory_lock); -+ -+ atomic_add(args->change_pages_nr, &session->mali_mem_allocated_pages); -+ if (atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > session->max_mali_mem_allocated_size) { -+ session->max_mali_mem_allocated_size = atomic_read(&session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args) -+{ -+ mali_mem_backend *mem_backend = NULL; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_PRINT(4, (" mali_mem_resize_memory called! \n")); -+ MALI_DEBUG_ASSERT(0 == args->psize % MALI_MMU_PAGE_SIZE); -+ -+ /* Get the memory backend that need to be resize. */ -+ mem_backend = mali_mem_backend_struct_search(session, args->vaddr); -+ -+ if (NULL == mem_backend) { -+ MALI_DEBUG_PRINT(2, ("_mali_ukk_mem_resize: memory backend = NULL!\n")); -+ return ret; -+ } -+ -+ MALI_DEBUG_ASSERT(args->psize != mem_backend->size); -+ -+ ret = mali_mem_resize(session, mem_backend, args->psize); -+ -+ return ret; -+} -+ -+_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args) -+{ -+ args->memory_usage = _mali_ukk_report_memory_usage(); -+ if (0 != args->vaddr) { -+ mali_mem_backend *mem_backend = NULL; -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ /* Get the backend that need to be modified. */ -+ mem_backend = mali_mem_backend_struct_search(session, args->vaddr); -+ if (NULL == mem_backend) { -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ if (MALI_MEM_COW == mem_backend->type) -+ args->change_pages_nr = mem_backend->cow_mem.change_pages_nr; -+ } -+ return _MALI_OSK_ERR_OK; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h -new file mode 100755 -index 000000000..23d8cde75 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_manager.h -@@ -0,0 +1,51 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_MANAGER_H__ -+#define __MALI_MEMORY_MANAGER_H__ -+ -+#include "mali_osk.h" -+#include -+#include -+#include -+#include -+#include -+#include "mali_memory_types.h" -+#include "mali_memory_os_alloc.h" -+#include "mali_uk_types.h" -+ -+struct mali_allocation_manager { -+ rwlock_t vm_lock; -+ struct rb_root allocation_mgr_rb; -+ struct list_head head; -+ struct mutex list_mutex; -+ u32 mali_allocation_num; -+}; -+ -+extern struct idr mali_backend_idr; -+extern struct mutex mali_idr_mutex; -+ -+int mali_memory_manager_init(struct mali_allocation_manager *mgr); -+void mali_memory_manager_uninit(struct mali_allocation_manager *mgr); -+ -+void mali_mem_allocation_struct_destory(mali_mem_allocation *alloc); -+_mali_osk_errcode_t mali_mem_add_mem_size(struct mali_session_data *session, u32 mali_addr, u32 add_size); -+mali_mem_backend *mali_mem_backend_struct_search(struct mali_session_data *session, u32 mali_address); -+_mali_osk_errcode_t _mali_ukk_mem_allocate(_mali_uk_alloc_mem_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_free(_mali_uk_free_mem_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_bind(_mali_uk_bind_mem_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_unbind(_mali_uk_unbind_mem_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_cow(_mali_uk_cow_mem_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_cow_modify_range(_mali_uk_cow_modify_range_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_usage_get(_mali_uk_profiling_memory_usage_get_s *args); -+_mali_osk_errcode_t _mali_ukk_mem_resize(_mali_uk_mem_resize_s *args); -+ -+#endif -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c -new file mode 100755 -index 000000000..1e1f5eb4a ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.c -@@ -0,0 +1,810 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "../platform/rk/custom_log.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_memory.h" -+#include "mali_memory_os_alloc.h" -+#include "mali_kernel_linux.h" -+ -+/* Minimum size of allocator page pool */ -+#define MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_MB * 256) -+#define MALI_OS_MEMORY_POOL_TRIM_JIFFIES (10 * CONFIG_HZ) /* Default to 10s */ -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) -+static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask); -+#else -+static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask); -+#endif -+#else -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); -+#else -+static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc); -+static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc); -+#endif -+#endif -+static void mali_mem_os_trim_pool(struct work_struct *work); -+ -+struct mali_mem_os_allocator mali_mem_os_allocator = { -+ .pool_lock = __SPIN_LOCK_UNLOCKED(pool_lock), -+ .pool_pages = LIST_HEAD_INIT(mali_mem_os_allocator.pool_pages), -+ .pool_count = 0, -+ -+ .allocated_pages = ATOMIC_INIT(0), -+ .allocation_limit = 0, -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+ .shrinker.shrink = mali_mem_os_shrink, -+#else -+ .shrinker.count_objects = mali_mem_os_shrink_count, -+ .shrinker.scan_objects = mali_mem_os_shrink, -+#endif -+ .shrinker.seeks = DEFAULT_SEEKS, -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) -+ .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool, TIMER_DEFERRABLE), -+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38) -+ .timed_shrinker = __DEFERRED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), -+#else -+ .timed_shrinker = __DELAYED_WORK_INITIALIZER(mali_mem_os_allocator.timed_shrinker, mali_mem_os_trim_pool), -+#endif -+}; -+ -+u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag) -+{ -+ LIST_HEAD(pages); -+ struct mali_page_node *m_page, *m_tmp; -+ u32 free_pages_nr = 0; -+ -+ if (MALI_TRUE == cow_flag) { -+ list_for_each_entry_safe(m_page, m_tmp, os_pages, list) { -+ /*only handle OS node here */ -+ if (m_page->type == MALI_PAGE_NODE_OS) { -+ if (1 == _mali_page_node_get_ref_count(m_page)) { -+ list_move(&m_page->list, &pages); -+ atomic_sub(1, &mali_mem_os_allocator.allocated_pages); -+ free_pages_nr ++; -+ } else { -+ _mali_page_node_unref(m_page); -+ m_page->page = NULL; -+ list_del(&m_page->list); -+ kfree(m_page); -+ } -+ } -+ } -+ } else { -+ list_cut_position(&pages, os_pages, os_pages->prev); -+ atomic_sub(pages_count, &mali_mem_os_allocator.allocated_pages); -+ free_pages_nr = pages_count; -+ } -+ -+ /* Put pages on pool. */ -+ spin_lock(&mali_mem_os_allocator.pool_lock); -+ list_splice(&pages, &mali_mem_os_allocator.pool_pages); -+ mali_mem_os_allocator.pool_count += free_pages_nr; -+ spin_unlock(&mali_mem_os_allocator.pool_lock); -+ -+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { -+ MALI_DEBUG_PRINT(5, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); -+ queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); -+ } -+ return free_pages_nr; -+} -+ -+/** -+* put page without put it into page pool -+*/ -+_mali_osk_errcode_t mali_mem_os_put_page(struct page *page) -+{ -+ MALI_DEBUG_ASSERT_POINTER(page); -+ if (1 == page_count(page)) { -+ atomic_sub(1, &mali_mem_os_allocator.allocated_pages); -+ dma_unmap_page(&mali_platform_device->dev, page_private(page), -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ ClearPagePrivate(page); -+ } -+ put_page(page); -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ u32 i = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem_from); -+ MALI_DEBUG_ASSERT_POINTER(mem_to); -+ -+ if (mem_from->count < start_page + page_count) { -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+ -+ list_for_each_entry_safe(m_page, m_tmp, &mem_from->pages, list) { -+ if (i >= start_page && i < start_page + page_count) { -+ list_move_tail(&m_page->list, &mem_to->pages); -+ mem_from->count--; -+ mem_to->count++; -+ } -+ i++; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size) -+{ -+ struct page *new_page; -+ LIST_HEAD(pages_list); -+ size_t page_count = PAGE_ALIGN(size) / _MALI_OSK_MALI_PAGE_SIZE; -+ size_t remaining = page_count; -+ struct mali_page_node *m_page, *m_tmp; -+ u32 i; -+ -+ MALI_DEBUG_ASSERT_POINTER(os_mem); -+ -+ if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { -+ MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", -+ size, -+ atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, -+ mali_mem_os_allocator.allocation_limit)); -+ return -ENOMEM; -+ } -+ -+ INIT_LIST_HEAD(&os_mem->pages); -+ os_mem->count = page_count; -+ -+ /* Grab pages from pool. */ -+ { -+ size_t pool_pages; -+ spin_lock(&mali_mem_os_allocator.pool_lock); -+ pool_pages = min(remaining, mali_mem_os_allocator.pool_count); -+ for (i = pool_pages; i > 0; i--) { -+ BUG_ON(list_empty(&mali_mem_os_allocator.pool_pages)); -+ list_move(mali_mem_os_allocator.pool_pages.next, &pages_list); -+ } -+ mali_mem_os_allocator.pool_count -= pool_pages; -+ remaining -= pool_pages; -+ spin_unlock(&mali_mem_os_allocator.pool_lock); -+ } -+ -+ /* Process pages from pool. */ -+ i = 0; -+ list_for_each_entry_safe(m_page, m_tmp, &pages_list, list) { -+ BUG_ON(NULL == m_page); -+ -+ list_move_tail(&m_page->list, &os_mem->pages); -+ } -+ -+ /* Allocate new pages, if needed. */ -+ for (i = 0; i < remaining; i++) { -+ dma_addr_t dma_addr; -+ gfp_t flags = __GFP_ZERO | GFP_HIGHUSER; -+ int err; -+ -+#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) -+ flags |= GFP_HIGHUSER; -+#else -+#ifdef CONFIG_ZONE_DMA32 -+ flags |= GFP_DMA32; -+#else -+#ifdef CONFIG_ZONE_DMA -+#else -+ /* arm64 utgard only work on < 4G, but the kernel -+ * didn't provide method to allocte memory < 4G -+ */ -+ MALI_DEBUG_ASSERT(0); -+#endif -+#endif -+#endif -+ -+ new_page = alloc_page(flags); -+ -+ if (unlikely(NULL == new_page)) { -+ E("err."); -+ /* Calculate the number of pages actually allocated, and free them. */ -+ os_mem->count = (page_count - remaining) + i; -+ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); -+ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); -+ return -ENOMEM; -+ } -+ -+ /* Ensure page is flushed from CPU caches. */ -+ dma_addr = dma_map_page(&mali_platform_device->dev, new_page, -+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ dma_unmap_page(&mali_platform_device->dev, dma_addr, -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ dma_addr = dma_map_page(&mali_platform_device->dev, new_page, -+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ -+ err = dma_mapping_error(&mali_platform_device->dev, dma_addr); -+ if (unlikely(err)) { -+ MALI_DEBUG_PRINT_ERROR(("OS Mem: Failed to DMA map page %p: %u", -+ new_page, err)); -+ __free_page(new_page); -+ os_mem->count = (page_count - remaining) + i; -+ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); -+ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); -+ return -EFAULT; -+ } -+ -+ /* Store page phys addr */ -+ SetPagePrivate(new_page); -+ set_page_private(new_page, dma_addr); -+ -+ m_page = _mali_page_node_allocate(MALI_PAGE_NODE_OS); -+ if (unlikely(NULL == m_page)) { -+ MALI_PRINT_ERROR(("OS Mem: Can't allocate mali_page node! \n")); -+ dma_unmap_page(&mali_platform_device->dev, page_private(new_page), -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ ClearPagePrivate(new_page); -+ __free_page(new_page); -+ os_mem->count = (page_count - remaining) + i; -+ atomic_add(os_mem->count, &mali_mem_os_allocator.allocated_pages); -+ mali_mem_os_free(&os_mem->pages, os_mem->count, MALI_FALSE); -+ return -EFAULT; -+ } -+ m_page->page = new_page; -+ -+ list_add_tail(&m_page->list, &os_mem->pages); -+ } -+ -+ atomic_add(page_count, &mali_mem_os_allocator.allocated_pages); -+ -+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { -+ MALI_DEBUG_PRINT(4, ("OS Mem: Stopping pool trim timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); -+ cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); -+ } -+ -+ return 0; -+} -+ -+ -+_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props) -+{ -+ struct mali_page_directory *pagedir = session->page_directory; -+ struct mali_page_node *m_page; -+ u32 virt; -+ u32 prop = props; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ MALI_DEBUG_ASSERT_POINTER(os_mem); -+ -+ MALI_DEBUG_ASSERT(start_page <= os_mem->count); -+ MALI_DEBUG_ASSERT((start_page + mapping_pgae_num) <= os_mem->count); -+ -+ if ((start_page + mapping_pgae_num) == os_mem->count) { -+ -+ virt = vaddr + MALI_MMU_PAGE_SIZE * (start_page + mapping_pgae_num); -+ -+ list_for_each_entry_reverse(m_page, &os_mem->pages, list) { -+ -+ virt -= MALI_MMU_PAGE_SIZE; -+ if (mapping_pgae_num > 0) { -+ dma_addr_t phys = page_private(m_page->page); -+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) -+ /* Verify that the "physical" address is 32-bit and -+ * usable for Mali, when on a system with bus addresses -+ * wider than 32-bit. */ -+ MALI_DEBUG_ASSERT(0 == (phys >> 32)); -+#endif -+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); -+ } else { -+ break; -+ } -+ mapping_pgae_num--; -+ } -+ -+ } else { -+ u32 i = 0; -+ virt = vaddr; -+ list_for_each_entry(m_page, &os_mem->pages, list) { -+ -+ if (i >= start_page) { -+ dma_addr_t phys = page_private(m_page->page); -+ -+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) -+ /* Verify that the "physical" address is 32-bit and -+ * usable for Mali, when on a system with bus addresses -+ * wider than 32-bit. */ -+ MALI_DEBUG_ASSERT(0 == (phys >> 32)); -+#endif -+ mali_mmu_pagedir_update(pagedir, virt, (mali_dma_addr)phys, MALI_MMU_PAGE_SIZE, prop); -+ } -+ i++; -+ virt += MALI_MMU_PAGE_SIZE; -+ } -+ } -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+void mali_mem_os_mali_unmap(mali_mem_allocation *alloc) -+{ -+ struct mali_session_data *session; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ mali_session_memory_lock(session); -+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ mali_session_memory_unlock(session); -+} -+ -+int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) -+{ -+ mali_mem_os_mem *os_mem = &mem_bkend->os_mem; -+ struct mali_page_node *m_page; -+ struct page *page; -+ int ret; -+ unsigned long addr = vma->vm_start; -+ MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); -+ -+ list_for_each_entry(m_page, &os_mem->pages, list) { -+ /* We should use vm_insert_page, but it does a dcache -+ * flush which makes it way slower than remap_pfn_range or vmf_insert_pfn. -+ ret = vm_insert_page(vma, addr, page); -+ */ -+ page = m_page->page; -+ ret = vmf_insert_pfn(vma, addr, page_to_pfn(page)); -+ -+ if (unlikely(0 != ret)) { -+ return -EFAULT; -+ } -+ addr += _MALI_OSK_MALI_PAGE_SIZE; -+ } -+ -+ return 0; -+} -+ -+_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size) -+{ -+ mali_mem_os_mem *os_mem = &mem_bkend->os_mem; -+ struct mali_page_node *m_page; -+ int ret; -+ int offset; -+ int mapping_page_num; -+ int count ; -+ -+ unsigned long vstart = vma->vm_start; -+ count = 0; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_OS); -+ MALI_DEBUG_ASSERT(0 == start_vaddr % _MALI_OSK_MALI_PAGE_SIZE); -+ MALI_DEBUG_ASSERT(0 == vstart % _MALI_OSK_MALI_PAGE_SIZE); -+ offset = (start_vaddr - vstart) / _MALI_OSK_MALI_PAGE_SIZE; -+ MALI_DEBUG_ASSERT(offset <= os_mem->count); -+ mapping_page_num = mappig_size / _MALI_OSK_MALI_PAGE_SIZE; -+ MALI_DEBUG_ASSERT((offset + mapping_page_num) <= os_mem->count); -+ -+ if ((offset + mapping_page_num) == os_mem->count) { -+ -+ unsigned long vm_end = start_vaddr + mappig_size; -+ -+ list_for_each_entry_reverse(m_page, &os_mem->pages, list) { -+ -+ vm_end -= _MALI_OSK_MALI_PAGE_SIZE; -+ if (mapping_page_num > 0) { -+ ret = vmf_insert_pfn(vma, vm_end, page_to_pfn(m_page->page)); -+ -+ if (unlikely(0 != ret)) { -+ /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ -+ if (-EBUSY == ret) { -+ break; -+ } else { -+ MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, offset is %d,page_count is %d\n", -+ ret, offset + mapping_page_num, os_mem->count)); -+ } -+ return _MALI_OSK_ERR_FAULT; -+ } -+ } else { -+ break; -+ } -+ mapping_page_num--; -+ -+ } -+ } else { -+ -+ list_for_each_entry(m_page, &os_mem->pages, list) { -+ if (count >= offset) { -+ -+ ret = vmf_insert_pfn(vma, vstart, page_to_pfn(m_page->page)); -+ -+ if (unlikely(0 != ret)) { -+ /*will return -EBUSY If the page has already been mapped into table, but it's OK*/ -+ if (-EBUSY == ret) { -+ break; -+ } else { -+ MALI_DEBUG_PRINT(1, ("OS Mem: mali_mem_os_resize_cpu_map_locked failed, ret = %d, count is %d, offset is %d,page_count is %d\n", -+ ret, count, offset, os_mem->count)); -+ } -+ return _MALI_OSK_ERR_FAULT; -+ } -+ } -+ count++; -+ vstart += _MALI_OSK_MALI_PAGE_SIZE; -+ } -+ } -+ return _MALI_OSK_ERR_OK; -+} -+ -+u32 mali_mem_os_release(mali_mem_backend *mem_bkend) -+{ -+ -+ mali_mem_allocation *alloc; -+ struct mali_session_data *session; -+ u32 free_pages_nr = 0; -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend); -+ MALI_DEBUG_ASSERT(MALI_MEM_OS == mem_bkend->type); -+ -+ alloc = mem_bkend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ /* Unmap the memory from the mali virtual address space. */ -+ mali_mem_os_mali_unmap(alloc); -+ mutex_lock(&mem_bkend->mutex); -+ /* Free pages */ -+ if (MALI_MEM_BACKEND_FLAG_COWED & mem_bkend->flags) { -+ /* Lock to avoid the free race condition for the cow shared memory page node. */ -+ _mali_osk_mutex_wait(session->cow_lock); -+ free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_TRUE); -+ _mali_osk_mutex_signal(session->cow_lock); -+ } else { -+ free_pages_nr = mali_mem_os_free(&mem_bkend->os_mem.pages, mem_bkend->os_mem.count, MALI_FALSE); -+ } -+ mutex_unlock(&mem_bkend->mutex); -+ -+ MALI_DEBUG_PRINT(4, ("OS Mem free : allocated size = 0x%x, free size = 0x%x\n", mem_bkend->os_mem.count * _MALI_OSK_MALI_PAGE_SIZE, -+ free_pages_nr * _MALI_OSK_MALI_PAGE_SIZE)); -+ -+ mem_bkend->os_mem.count = 0; -+ return free_pages_nr; -+} -+ -+ -+#define MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE 128 -+static struct { -+ struct { -+ mali_dma_addr phys; -+ mali_io_address mapping; -+ } page[MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE]; -+ size_t count; -+ spinlock_t lock; -+} mali_mem_page_table_page_pool = { -+ .count = 0, -+ .lock = __SPIN_LOCK_UNLOCKED(pool_lock), -+}; -+ -+_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping) -+{ -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_NOMEM; -+ dma_addr_t tmp_phys; -+ -+ spin_lock(&mali_mem_page_table_page_pool.lock); -+ if (0 < mali_mem_page_table_page_pool.count) { -+ u32 i = --mali_mem_page_table_page_pool.count; -+ *phys = mali_mem_page_table_page_pool.page[i].phys; -+ *mapping = mali_mem_page_table_page_pool.page[i].mapping; -+ -+ ret = _MALI_OSK_ERR_OK; -+ } -+ spin_unlock(&mali_mem_page_table_page_pool.lock); -+ -+ if (_MALI_OSK_ERR_OK != ret) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) -+ *mapping = dma_alloc_attrs(&mali_platform_device->dev, -+ _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, -+ GFP_KERNEL, DMA_ATTR_WRITE_COMBINE); -+#else -+ *mapping = dma_alloc_writecombine(&mali_platform_device->dev, -+ _MALI_OSK_MALI_PAGE_SIZE, &tmp_phys, GFP_KERNEL); -+#endif -+ if (NULL != *mapping) { -+ ret = _MALI_OSK_ERR_OK; -+ -+#if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) -+ /* Verify that the "physical" address is 32-bit and -+ * usable for Mali, when on a system with bus addresses -+ * wider than 32-bit. */ -+ MALI_DEBUG_ASSERT(0 == (tmp_phys >> 32)); -+#endif -+ -+ *phys = (mali_dma_addr)tmp_phys; -+ } -+ } -+ -+ return ret; -+} -+ -+void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt) -+{ -+ spin_lock(&mali_mem_page_table_page_pool.lock); -+ if (MALI_MEM_OS_PAGE_TABLE_PAGE_POOL_SIZE > mali_mem_page_table_page_pool.count) { -+ u32 i = mali_mem_page_table_page_pool.count; -+ mali_mem_page_table_page_pool.page[i].phys = phys; -+ mali_mem_page_table_page_pool.page[i].mapping = virt; -+ -+ ++mali_mem_page_table_page_pool.count; -+ -+ spin_unlock(&mali_mem_page_table_page_pool.lock); -+ } else { -+ spin_unlock(&mali_mem_page_table_page_pool.lock); -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) -+ dma_free_attrs(&mali_platform_device->dev, -+ _MALI_OSK_MALI_PAGE_SIZE, virt, phys, -+ DMA_ATTR_WRITE_COMBINE); -+#else -+ dma_free_writecombine(&mali_platform_device->dev, -+ _MALI_OSK_MALI_PAGE_SIZE, virt, phys); -+#endif -+ } -+} -+ -+void mali_mem_os_free_page_node(struct mali_page_node *m_page) -+{ -+ struct page *page = m_page->page; -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_OS); -+ -+ if (1 == page_count(page)) { -+ dma_unmap_page(&mali_platform_device->dev, page_private(page), -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_BIDIRECTIONAL); -+ ClearPagePrivate(page); -+ } -+ __free_page(page); -+ m_page->page = NULL; -+ list_del(&m_page->list); -+ kfree(m_page); -+} -+ -+/* The maximum number of page table pool pages to free in one go. */ -+#define MALI_MEM_OS_CHUNK_TO_FREE 64UL -+ -+/* Free a certain number of pages from the page table page pool. -+ * The pool lock must be held when calling the function, and the lock will be -+ * released before returning. -+ */ -+static void mali_mem_os_page_table_pool_free(size_t nr_to_free) -+{ -+ mali_dma_addr phys_arr[MALI_MEM_OS_CHUNK_TO_FREE]; -+ void *virt_arr[MALI_MEM_OS_CHUNK_TO_FREE]; -+ u32 i; -+ -+ MALI_DEBUG_ASSERT(nr_to_free <= MALI_MEM_OS_CHUNK_TO_FREE); -+ -+ /* Remove nr_to_free pages from the pool and store them locally on stack. */ -+ for (i = 0; i < nr_to_free; i++) { -+ u32 pool_index = mali_mem_page_table_page_pool.count - i - 1; -+ -+ phys_arr[i] = mali_mem_page_table_page_pool.page[pool_index].phys; -+ virt_arr[i] = mali_mem_page_table_page_pool.page[pool_index].mapping; -+ } -+ -+ mali_mem_page_table_page_pool.count -= nr_to_free; -+ -+ spin_unlock(&mali_mem_page_table_page_pool.lock); -+ -+ /* After releasing the spinlock: free the pages we removed from the pool. */ -+ for (i = 0; i < nr_to_free; i++) { -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) -+ dma_free_attrs(&mali_platform_device->dev, _MALI_OSK_MALI_PAGE_SIZE, -+ virt_arr[i], (dma_addr_t)phys_arr[i], -+ DMA_ATTR_WRITE_COMBINE); -+#else -+ dma_free_writecombine(&mali_platform_device->dev, -+ _MALI_OSK_MALI_PAGE_SIZE, -+ virt_arr[i], (dma_addr_t)phys_arr[i]); -+#endif -+ } -+} -+ -+static void mali_mem_os_trim_page_table_page_pool(void) -+{ -+ size_t nr_to_free = 0; -+ size_t nr_to_keep; -+ -+ /* Keep 2 page table pages for each 1024 pages in the page cache. */ -+ nr_to_keep = mali_mem_os_allocator.pool_count / 512; -+ /* And a minimum of eight pages, to accomodate new sessions. */ -+ nr_to_keep += 8; -+ -+ if (0 == spin_trylock(&mali_mem_page_table_page_pool.lock)) return; -+ -+ if (nr_to_keep < mali_mem_page_table_page_pool.count) { -+ nr_to_free = mali_mem_page_table_page_pool.count - nr_to_keep; -+ nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, nr_to_free); -+ } -+ -+ /* Pool lock will be released by the callee. */ -+ mali_mem_os_page_table_pool_free(nr_to_free); -+} -+ -+static unsigned long mali_mem_os_shrink_count(struct shrinker *shrinker, struct shrink_control *sc) -+{ -+ return mali_mem_os_allocator.pool_count; -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) -+static int mali_mem_os_shrink(int nr_to_scan, gfp_t gfp_mask) -+#else -+static int mali_mem_os_shrink(struct shrinker *shrinker, int nr_to_scan, gfp_t gfp_mask) -+#endif /* Linux < 2.6.35 */ -+#else -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+static int mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) -+#else -+static unsigned long mali_mem_os_shrink(struct shrinker *shrinker, struct shrink_control *sc) -+#endif /* Linux < 3.12.0 */ -+#endif /* Linux < 3.0.0 */ -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ unsigned long flags; -+ struct list_head *le, pages; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0) -+ int nr = nr_to_scan; -+#else -+ int nr = sc->nr_to_scan; -+#endif -+ -+ if (0 == nr) { -+ return mali_mem_os_shrink_count(shrinker, sc); -+ } -+ -+ if (0 == spin_trylock_irqsave(&mali_mem_os_allocator.pool_lock, flags)) { -+ /* Not able to lock. */ -+ return -1; -+ } -+ -+ if (0 == mali_mem_os_allocator.pool_count) { -+ /* No pages availble */ -+ spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); -+ return 0; -+ } -+ -+ /* Release from general page pool */ -+ nr = min((size_t)nr, mali_mem_os_allocator.pool_count); -+ mali_mem_os_allocator.pool_count -= nr; -+ list_for_each(le, &mali_mem_os_allocator.pool_pages) { -+ --nr; -+ if (0 == nr) break; -+ } -+ list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); -+ spin_unlock_irqrestore(&mali_mem_os_allocator.pool_lock, flags); -+ -+ list_for_each_entry_safe(m_page, m_tmp, &pages, list) { -+ mali_mem_os_free_page_node(m_page); -+ } -+ -+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES > mali_mem_os_allocator.pool_count) { -+ /* Pools are empty, stop timer */ -+ MALI_DEBUG_PRINT(5, ("Stopping timer, only %u pages on pool\n", mali_mem_os_allocator.pool_count)); -+ cancel_delayed_work(&mali_mem_os_allocator.timed_shrinker); -+ } -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+ return mali_mem_os_shrink_count(shrinker, sc); -+#else -+ return nr; -+#endif -+} -+ -+static void mali_mem_os_trim_pool(struct work_struct *data) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ struct list_head *le; -+ LIST_HEAD(pages); -+ size_t nr_to_free; -+ -+ MALI_IGNORE(data); -+ -+ MALI_DEBUG_PRINT(3, ("OS Mem: Trimming pool %u\n", mali_mem_os_allocator.pool_count)); -+ -+ /* Release from general page pool */ -+ spin_lock(&mali_mem_os_allocator.pool_lock); -+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { -+ size_t count = mali_mem_os_allocator.pool_count - MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES; -+ const size_t min_to_free = min(64, MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES); -+ -+ /* Free half the pages on the pool above the static limit. Or 64 pages, 256KB. */ -+ nr_to_free = max(count / 2, min_to_free); -+ -+ mali_mem_os_allocator.pool_count -= nr_to_free; -+ list_for_each(le, &mali_mem_os_allocator.pool_pages) { -+ --nr_to_free; -+ if (0 == nr_to_free) break; -+ } -+ list_cut_position(&pages, &mali_mem_os_allocator.pool_pages, le); -+ } -+ spin_unlock(&mali_mem_os_allocator.pool_lock); -+ -+ list_for_each_entry_safe(m_page, m_tmp, &pages, list) { -+ mali_mem_os_free_page_node(m_page); -+ } -+ -+ /* Release some pages from page table page pool */ -+ mali_mem_os_trim_page_table_page_pool(); -+ -+ if (MALI_OS_MEMORY_KERNEL_BUFFER_SIZE_IN_PAGES < mali_mem_os_allocator.pool_count) { -+ MALI_DEBUG_PRINT(4, ("OS Mem: Starting pool trim timer %u\n", mali_mem_os_allocator.pool_count)); -+ queue_delayed_work(mali_mem_os_allocator.wq, &mali_mem_os_allocator.timed_shrinker, MALI_OS_MEMORY_POOL_TRIM_JIFFIES); -+ } -+} -+ -+_mali_osk_errcode_t mali_mem_os_init(void) -+{ -+ mali_mem_os_allocator.wq = alloc_workqueue("mali-mem", WQ_UNBOUND, 1); -+ if (NULL == mali_mem_os_allocator.wq) { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ register_shrinker(&mali_mem_os_allocator.shrinker); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void mali_mem_os_term(void) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ unregister_shrinker(&mali_mem_os_allocator.shrinker); -+ cancel_delayed_work_sync(&mali_mem_os_allocator.timed_shrinker); -+ -+ if (NULL != mali_mem_os_allocator.wq) { -+ destroy_workqueue(mali_mem_os_allocator.wq); -+ mali_mem_os_allocator.wq = NULL; -+ } -+ -+ spin_lock(&mali_mem_os_allocator.pool_lock); -+ list_for_each_entry_safe(m_page, m_tmp, &mali_mem_os_allocator.pool_pages, list) { -+ mali_mem_os_free_page_node(m_page); -+ -+ --mali_mem_os_allocator.pool_count; -+ } -+ BUG_ON(mali_mem_os_allocator.pool_count); -+ spin_unlock(&mali_mem_os_allocator.pool_lock); -+ -+ /* Release from page table page pool */ -+ do { -+ u32 nr_to_free; -+ -+ spin_lock(&mali_mem_page_table_page_pool.lock); -+ -+ nr_to_free = min((size_t)MALI_MEM_OS_CHUNK_TO_FREE, mali_mem_page_table_page_pool.count); -+ -+ /* Pool lock will be released by the callee. */ -+ mali_mem_os_page_table_pool_free(nr_to_free); -+ } while (0 != mali_mem_page_table_page_pool.count); -+} -+ -+_mali_osk_errcode_t mali_memory_core_resource_os_memory(u32 size) -+{ -+ mali_mem_os_allocator.allocation_limit = size; -+ -+ MALI_SUCCESS; -+} -+ -+u32 mali_mem_os_stat(void) -+{ -+ return atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h -new file mode 100755 -index 000000000..8c9b35d0b ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_os_alloc.h -@@ -0,0 +1,54 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_OS_ALLOC_H__ -+#define __MALI_MEMORY_OS_ALLOC_H__ -+ -+#include "mali_osk.h" -+#include "mali_memory_types.h" -+ -+ -+/** @brief Release Mali OS memory -+ * -+ * The session memory_lock must be held when calling this function. -+ * -+ * @param mem_bkend Pointer to the mali_mem_backend to release -+ */ -+u32 mali_mem_os_release(mali_mem_backend *mem_bkend); -+ -+_mali_osk_errcode_t mali_mem_os_get_table_page(mali_dma_addr *phys, mali_io_address *mapping); -+ -+void mali_mem_os_release_table_page(mali_dma_addr phys, void *virt); -+ -+_mali_osk_errcode_t mali_mem_os_init(void); -+ -+void mali_mem_os_term(void); -+ -+u32 mali_mem_os_stat(void); -+ -+void mali_mem_os_free_page_node(struct mali_page_node *m_page); -+ -+int mali_mem_os_alloc_pages(mali_mem_os_mem *os_mem, u32 size); -+ -+u32 mali_mem_os_free(struct list_head *os_pages, u32 pages_count, mali_bool cow_flag); -+ -+_mali_osk_errcode_t mali_mem_os_put_page(struct page *page); -+ -+_mali_osk_errcode_t mali_mem_os_resize_pages(mali_mem_os_mem *mem_from, mali_mem_os_mem *mem_to, u32 start_page, u32 page_count); -+ -+_mali_osk_errcode_t mali_mem_os_mali_map(mali_mem_os_mem *os_mem, struct mali_session_data *session, u32 vaddr, u32 start_page, u32 mapping_pgae_num, u32 props); -+ -+void mali_mem_os_mali_unmap(mali_mem_allocation *alloc); -+ -+int mali_mem_os_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); -+ -+_mali_osk_errcode_t mali_mem_os_resize_cpu_map_locked(mali_mem_backend *mem_bkend, struct vm_area_struct *vma, unsigned long start_vaddr, u32 mappig_size); -+ -+#endif /* __MALI_MEMORY_OS_ALLOC_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c -new file mode 100755 -index 000000000..0b4f82868 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.c -@@ -0,0 +1,170 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_memory.h" -+#include "mali_memory_secure.h" -+#include "mali_osk.h" -+#include -+#include -+#include -+#include -+ -+_mali_osk_errcode_t mali_mem_secure_attach_dma_buf(mali_mem_secure *secure_mem, u32 size, int mem_fd) -+{ -+ struct dma_buf *buf; -+ MALI_DEBUG_ASSERT_POINTER(secure_mem); -+ -+ /* get dma buffer */ -+ buf = dma_buf_get(mem_fd); -+ if (IS_ERR_OR_NULL(buf)) { -+ MALI_DEBUG_PRINT_ERROR(("Failed to get dma buf!\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ if (size != buf->size) { -+ MALI_DEBUG_PRINT_ERROR(("The secure mem size not match to the dma buf size!\n")); -+ goto failed_alloc_mem; -+ } -+ -+ secure_mem->buf = buf; -+ secure_mem->attachment = dma_buf_attach(secure_mem->buf, &mali_platform_device->dev); -+ if (NULL == secure_mem->attachment) { -+ MALI_DEBUG_PRINT_ERROR(("Failed to get dma buf attachment!\n")); -+ goto failed_dma_attach; -+ } -+ -+ secure_mem->sgt = dma_buf_map_attachment(secure_mem->attachment, DMA_BIDIRECTIONAL); -+ if (IS_ERR_OR_NULL(secure_mem->sgt)) { -+ MALI_DEBUG_PRINT_ERROR(("Failed to map dma buf attachment\n")); -+ goto failed_dma_map; -+ } -+ -+ secure_mem->count = size / MALI_MMU_PAGE_SIZE; -+ -+ return _MALI_OSK_ERR_OK; -+ -+failed_dma_map: -+ dma_buf_detach(secure_mem->buf, secure_mem->attachment); -+failed_dma_attach: -+failed_alloc_mem: -+ dma_buf_put(buf); -+ return _MALI_OSK_ERR_FAULT; -+} -+ -+_mali_osk_errcode_t mali_mem_secure_mali_map(mali_mem_secure *secure_mem, struct mali_session_data *session, u32 vaddr, u32 props) -+{ -+ struct mali_page_directory *pagedir; -+ struct scatterlist *sg; -+ u32 virt = vaddr; -+ u32 prop = props; -+ int i; -+ -+ MALI_DEBUG_ASSERT_POINTER(secure_mem); -+ MALI_DEBUG_ASSERT_POINTER(secure_mem->sgt); -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ pagedir = session->page_directory; -+ -+ for_each_sg(secure_mem->sgt->sgl, sg, secure_mem->sgt->nents, i) { -+ u32 size = sg_dma_len(sg); -+ dma_addr_t phys = sg_dma_address(sg); -+ -+ /* sg must be page aligned. */ -+ MALI_DEBUG_ASSERT(0 == size % MALI_MMU_PAGE_SIZE); -+ MALI_DEBUG_ASSERT(0 == (phys & ~(uintptr_t)0xFFFFFFFF)); -+ -+ mali_mmu_pagedir_update(pagedir, virt, phys, size, prop); -+ -+ MALI_DEBUG_PRINT(3, ("The secure mem physical address: 0x%x gpu virtual address: 0x%x! \n", phys, virt)); -+ virt += size; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void mali_mem_secure_mali_unmap(mali_mem_allocation *alloc) -+{ -+ struct mali_session_data *session; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ mali_session_memory_lock(session); -+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ mali_session_memory_unlock(session); -+} -+ -+ -+int mali_mem_secure_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma) -+{ -+ -+ int ret = 0; -+ struct scatterlist *sg; -+ mali_mem_secure *secure_mem = &mem_bkend->secure_mem; -+ unsigned long addr = vma->vm_start; -+ int i; -+ -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_SECURE); -+ -+ for_each_sg(secure_mem->sgt->sgl, sg, secure_mem->sgt->nents, i) { -+ phys_addr_t phys; -+ dma_addr_t dev_addr; -+ u32 size, j; -+ dev_addr = sg_dma_address(sg); -+#if defined(CONFIG_ARM64) ||LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) -+ phys = dma_to_phys(&mali_platform_device->dev, dev_addr); -+#else -+ phys = page_to_phys(pfn_to_page(dma_to_pfn(&mali_platform_device->dev, dev_addr))); -+#endif -+ size = sg_dma_len(sg); -+ MALI_DEBUG_ASSERT(0 == size % _MALI_OSK_MALI_PAGE_SIZE); -+ -+ for (j = 0; j < size / _MALI_OSK_MALI_PAGE_SIZE; j++) { -+ ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys)); -+ -+ if (unlikely(0 != ret)) { -+ return -EFAULT; -+ } -+ addr += _MALI_OSK_MALI_PAGE_SIZE; -+ phys += _MALI_OSK_MALI_PAGE_SIZE; -+ -+ MALI_DEBUG_PRINT(3, ("The secure mem physical address: 0x%x , cpu virtual address: 0x%x! \n", phys, addr)); -+ } -+ } -+ return ret; -+} -+ -+u32 mali_mem_secure_release(mali_mem_backend *mem_bkend) -+{ -+ struct mali_mem_secure *mem; -+ mali_mem_allocation *alloc = mem_bkend->mali_allocation; -+ u32 free_pages_nr = 0; -+ MALI_DEBUG_ASSERT(mem_bkend->type == MALI_MEM_SECURE); -+ -+ mem = &mem_bkend->secure_mem; -+ MALI_DEBUG_ASSERT_POINTER(mem->attachment); -+ MALI_DEBUG_ASSERT_POINTER(mem->buf); -+ MALI_DEBUG_ASSERT_POINTER(mem->sgt); -+ /* Unmap the memory from the mali virtual address space. */ -+ mali_mem_secure_mali_unmap(alloc); -+ mutex_lock(&mem_bkend->mutex); -+ dma_buf_unmap_attachment(mem->attachment, mem->sgt, DMA_BIDIRECTIONAL); -+ dma_buf_detach(mem->buf, mem->attachment); -+ dma_buf_put(mem->buf); -+ mutex_unlock(&mem_bkend->mutex); -+ -+ free_pages_nr = mem->count; -+ -+ return free_pages_nr; -+} -+ -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h -new file mode 100755 -index 000000000..48691d479 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_secure.h -@@ -0,0 +1,30 @@ -+/* -+ * Copyright (C) 2010, 2013, 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_SECURE_H__ -+#define __MALI_MEMORY_SECURE_H__ -+ -+#include "mali_session.h" -+#include "mali_memory.h" -+#include -+ -+#include "mali_memory_types.h" -+ -+_mali_osk_errcode_t mali_mem_secure_attach_dma_buf(mali_mem_secure *secure_mem, u32 size, int mem_fd); -+ -+_mali_osk_errcode_t mali_mem_secure_mali_map(mali_mem_secure *secure_mem, struct mali_session_data *session, u32 vaddr, u32 props); -+ -+void mali_mem_secure_mali_unmap(mali_mem_allocation *alloc); -+ -+int mali_mem_secure_cpu_map(mali_mem_backend *mem_bkend, struct vm_area_struct *vma); -+ -+u32 mali_mem_secure_release(mali_mem_backend *mem_bkend); -+ -+#endif /* __MALI_MEMORY_SECURE_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c -new file mode 100755 -index 000000000..d682785b9 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.c -@@ -0,0 +1,943 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "mali_osk.h" -+#include "mali_osk_mali.h" -+#include "mali_memory.h" -+#include "mali_memory_manager.h" -+#include "mali_memory_virtual.h" -+#include "mali_memory_cow.h" -+#include "mali_ukk.h" -+#include "mali_kernel_utilization.h" -+#include "mali_memory_swap_alloc.h" -+ -+ -+static struct _mali_osk_bitmap idx_mgr; -+static struct file *global_swap_file; -+static struct address_space *global_swap_space; -+static _mali_osk_wq_work_t *mali_mem_swap_out_workq = NULL; -+static u32 mem_backend_swapped_pool_size; -+#ifdef MALI_MEM_SWAP_TRACKING -+static u32 mem_backend_swapped_unlock_size; -+#endif -+/* Lock order: mem_backend_swapped_pool_lock > each memory backend's mutex lock. -+ * This lock used to protect mem_backend_swapped_pool_size and mem_backend_swapped_pool. */ -+static struct mutex mem_backend_swapped_pool_lock; -+static struct list_head mem_backend_swapped_pool; -+ -+extern struct mali_mem_os_allocator mali_mem_os_allocator; -+ -+#define MALI_SWAP_LOW_MEM_DEFAULT_VALUE (60*1024*1024) -+#define MALI_SWAP_INVALIDATE_MALI_ADDRESS (0) /* Used to mark the given memory cookie is invalidate. */ -+#define MALI_SWAP_GLOBAL_SWAP_FILE_SIZE (0xFFFFFFFF) -+#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX \ -+ ((MALI_SWAP_GLOBAL_SWAP_FILE_SIZE) >> PAGE_SHIFT) -+#define MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE (1 << 15) /* Reserved for CoW nonlinear swap backend memory, the space size is 128MB. */ -+ -+unsigned int mali_mem_swap_out_threshold_value = MALI_SWAP_LOW_MEM_DEFAULT_VALUE; -+ -+/** -+ * We have two situations to do shrinking things, one is we met low GPU utilization which shows GPU needn't touch too -+ * swappable backends in short time, and the other one is we add new swappable backends, the total pool size exceed -+ * the threshold value of the swapped pool size. -+ */ -+typedef enum { -+ MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION = 100, -+ MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS = 257, -+} _mali_mem_swap_pool_shrink_type_t; -+ -+static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg); -+ -+_mali_osk_errcode_t mali_mem_swap_init(void) -+{ -+ gfp_t flags = __GFP_NORETRY | __GFP_NOWARN; -+ -+ if (_MALI_OSK_ERR_OK != _mali_osk_bitmap_init(&idx_mgr, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX, MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE)) { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ global_swap_file = shmem_file_setup("mali_swap", MALI_SWAP_GLOBAL_SWAP_FILE_SIZE, VM_NORESERVE); -+ if (IS_ERR(global_swap_file)) { -+ _mali_osk_bitmap_term(&idx_mgr); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ global_swap_space = global_swap_file->f_path.dentry->d_inode->i_mapping; -+ -+ mali_mem_swap_out_workq = _mali_osk_wq_create_work(mali_mem_swap_swapped_bkend_pool_check_for_low_utilization, NULL); -+ if (NULL == mali_mem_swap_out_workq) { -+ _mali_osk_bitmap_term(&idx_mgr); -+ fput(global_swap_file); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+#if defined(CONFIG_ARM) && !defined(CONFIG_ARM_LPAE) -+ flags |= GFP_HIGHUSER; -+#else -+#ifdef CONFIG_ZONE_DMA32 -+ flags |= GFP_DMA32; -+#else -+#ifdef CONFIG_ZONE_DMA -+ flags |= GFP_DMA; -+#else -+ /* arm64 utgard only work on < 4G, but the kernel -+ * didn't provide method to allocte memory < 4G -+ */ -+ MALI_DEBUG_ASSERT(0); -+#endif -+#endif -+#endif -+ -+ /* When we use shmem_read_mapping_page to allocate/swap-in, it will -+ * use these flags to allocate new page if need.*/ -+ mapping_set_gfp_mask(global_swap_space, flags); -+ -+ mem_backend_swapped_pool_size = 0; -+#ifdef MALI_MEM_SWAP_TRACKING -+ mem_backend_swapped_unlock_size = 0; -+#endif -+ mutex_init(&mem_backend_swapped_pool_lock); -+ INIT_LIST_HEAD(&mem_backend_swapped_pool); -+ -+ MALI_DEBUG_PRINT(2, ("Mali SWAP: Swap out threshold vaule is %uM\n", mali_mem_swap_out_threshold_value >> 20)); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void mali_mem_swap_term(void) -+{ -+ _mali_osk_bitmap_term(&idx_mgr); -+ -+ fput(global_swap_file); -+ -+ _mali_osk_wq_delete_work(mali_mem_swap_out_workq); -+ -+ MALI_DEBUG_ASSERT(list_empty(&mem_backend_swapped_pool)); -+ MALI_DEBUG_ASSERT(0 == mem_backend_swapped_pool_size); -+ -+ return; -+} -+ -+struct file *mali_mem_swap_get_global_swap_file(void) -+{ -+ return global_swap_file; -+} -+ -+/* Judge if swappable backend in swapped pool. */ -+static mali_bool mali_memory_swap_backend_in_swapped_pool(mali_mem_backend *mem_bkend) -+{ -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend); -+ -+ return !list_empty(&mem_bkend->list); -+} -+ -+void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend) -+{ -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend); -+ -+ mutex_lock(&mem_backend_swapped_pool_lock); -+ mutex_lock(&mem_bkend->mutex); -+ -+ if (MALI_FALSE == mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { -+ mutex_unlock(&mem_bkend->mutex); -+ mutex_unlock(&mem_backend_swapped_pool_lock); -+ return; -+ } -+ -+ MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); -+ -+ list_del_init(&mem_bkend->list); -+ -+ mutex_unlock(&mem_bkend->mutex); -+ -+ mem_backend_swapped_pool_size -= mem_bkend->size; -+ -+ mutex_unlock(&mem_backend_swapped_pool_lock); -+} -+ -+static void mali_mem_swap_out_page_node(mali_page_node *page_node) -+{ -+ MALI_DEBUG_ASSERT(page_node); -+ -+ dma_unmap_page(&mali_platform_device->dev, page_node->swap_it->dma_addr, -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); -+ set_page_dirty(page_node->swap_it->page); -+ put_page(page_node->swap_it->page); -+} -+ -+void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend) -+{ -+ mali_page_node *m_page; -+ -+ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); -+ -+ if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN)) { -+ return; -+ } -+ -+ mem_bkend->flags |= MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN; -+ -+ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { -+ mali_mem_swap_out_page_node(m_page); -+ } -+ -+ return; -+} -+ -+static void mali_mem_swap_unlock_partial_locked_mem_backend(mali_mem_backend *mem_bkend, mali_page_node *page_node) -+{ -+ mali_page_node *m_page; -+ -+ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_bkend->mutex)); -+ -+ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { -+ if (m_page == page_node) { -+ break; -+ } -+ mali_mem_swap_out_page_node(m_page); -+ } -+} -+ -+static void mali_mem_swap_swapped_bkend_pool_shrink(_mali_mem_swap_pool_shrink_type_t shrink_type) -+{ -+ mali_mem_backend *bkend, *tmp_bkend; -+ long system_free_size; -+ u32 last_gpu_utilization, gpu_utilization_threshold_value, temp_swap_out_threshold_value; -+ -+ MALI_DEBUG_ASSERT(1 == mutex_is_locked(&mem_backend_swapped_pool_lock)); -+ -+ if (MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION == shrink_type) { -+ /** -+ * When we met that system memory is very low and Mali locked swappable memory size is less than -+ * threshold value, and at the same time, GPU load is very low and don't need high performance, -+ * at this condition, we can unlock more swap memory backend from swapped backends pool. -+ */ -+ gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION; -+ temp_swap_out_threshold_value = (mali_mem_swap_out_threshold_value >> 2); -+ } else { -+ /* When we add swappable memory backends to swapped pool, we need to think that we couldn't -+ * hold too much swappable backends in Mali driver, and also we need considering performance. -+ * So there is a balance for swapping out memory backend, we should follow the following conditions: -+ * 1. Total memory size in global mem backend swapped pool is more than the defined threshold value. -+ * 2. System level free memory size is less than the defined threshold value. -+ * 3. Please note that GPU utilization problem isn't considered in this condition. -+ */ -+ gpu_utilization_threshold_value = MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS; -+ temp_swap_out_threshold_value = mali_mem_swap_out_threshold_value; -+ } -+ -+ /* Get system free pages number. */ -+ system_free_size = global_zone_page_state(NR_FREE_PAGES) * PAGE_SIZE; -+ last_gpu_utilization = _mali_ukk_utilization_gp_pp(); -+ -+ if ((last_gpu_utilization < gpu_utilization_threshold_value) -+ && (system_free_size < mali_mem_swap_out_threshold_value) -+ && (mem_backend_swapped_pool_size > temp_swap_out_threshold_value)) { -+ list_for_each_entry_safe(bkend, tmp_bkend, &mem_backend_swapped_pool, list) { -+ if (mem_backend_swapped_pool_size <= temp_swap_out_threshold_value) { -+ break; -+ } -+ -+ mutex_lock(&bkend->mutex); -+ -+ /* check if backend is in use. */ -+ if (0 < bkend->using_count) { -+ mutex_unlock(&bkend->mutex); -+ continue; -+ } -+ -+ mali_mem_swap_unlock_single_mem_backend(bkend); -+ list_del_init(&bkend->list); -+ mem_backend_swapped_pool_size -= bkend->size; -+#ifdef MALI_MEM_SWAP_TRACKING -+ mem_backend_swapped_unlock_size += bkend->size; -+#endif -+ mutex_unlock(&bkend->mutex); -+ } -+ } -+ -+ return; -+} -+ -+static void mali_mem_swap_swapped_bkend_pool_check_for_low_utilization(void *arg) -+{ -+ MALI_IGNORE(arg); -+ -+ mutex_lock(&mem_backend_swapped_pool_lock); -+ -+ mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_WITH_LOW_UTILIZATION); -+ -+ mutex_unlock(&mem_backend_swapped_pool_lock); -+} -+ -+/** -+ * After PP job finished, we add all of swappable memory backend used by this PP -+ * job to the tail of the global swapped pool, and if the total size of swappable memory is more than threshold -+ * value, we also need to shrink the swapped pool start from the head of the list. -+ */ -+void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend) -+{ -+ mutex_lock(&mem_backend_swapped_pool_lock); -+ mutex_lock(&mem_bkend->mutex); -+ -+ if (mali_memory_swap_backend_in_swapped_pool(mem_bkend)) { -+ MALI_DEBUG_ASSERT(!list_empty(&mem_bkend->list)); -+ -+ list_del_init(&mem_bkend->list); -+ list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); -+ mutex_unlock(&mem_bkend->mutex); -+ mutex_unlock(&mem_backend_swapped_pool_lock); -+ return; -+ } -+ -+ list_add_tail(&mem_bkend->list, &mem_backend_swapped_pool); -+ -+ mutex_unlock(&mem_bkend->mutex); -+ mem_backend_swapped_pool_size += mem_bkend->size; -+ -+ mali_mem_swap_swapped_bkend_pool_shrink(MALI_MEM_SWAP_SHRINK_FOR_ADDING_NEW_BACKENDS); -+ -+ mutex_unlock(&mem_backend_swapped_pool_lock); -+ return; -+} -+ -+ -+u32 mali_mem_swap_idx_alloc(void) -+{ -+ return _mali_osk_bitmap_alloc(&idx_mgr); -+} -+ -+void mali_mem_swap_idx_free(u32 idx) -+{ -+ _mali_osk_bitmap_free(&idx_mgr, idx); -+} -+ -+static u32 mali_mem_swap_idx_range_alloc(u32 count) -+{ -+ u32 index; -+ -+ index = _mali_osk_bitmap_alloc_range(&idx_mgr, count); -+ -+ return index; -+} -+ -+static void mali_mem_swap_idx_range_free(u32 idx, int num) -+{ -+ _mali_osk_bitmap_free_range(&idx_mgr, idx, num); -+} -+ -+struct mali_swap_item *mali_mem_swap_alloc_swap_item(void) -+{ -+ mali_swap_item *swap_item; -+ -+ swap_item = kzalloc(sizeof(mali_swap_item), GFP_KERNEL); -+ -+ if (NULL == swap_item) { -+ return NULL; -+ } -+ -+ atomic_set(&swap_item->ref_count, 1); -+ swap_item->page = NULL; -+ atomic_add(1, &mali_mem_os_allocator.allocated_pages); -+ -+ return swap_item; -+} -+ -+void mali_mem_swap_free_swap_item(mali_swap_item *swap_item) -+{ -+ struct inode *file_node; -+ long long start, end; -+ -+ /* If this swap item is shared, we just reduce the reference counter. */ -+ if (0 == atomic_dec_return(&swap_item->ref_count)) { -+ file_node = global_swap_file->f_path.dentry->d_inode; -+ start = swap_item->idx; -+ start = start << 12; -+ end = start + PAGE_SIZE; -+ -+ shmem_truncate_range(file_node, start, (end - 1)); -+ -+ mali_mem_swap_idx_free(swap_item->idx); -+ -+ atomic_sub(1, &mali_mem_os_allocator.allocated_pages); -+ -+ kfree(swap_item); -+ } -+} -+ -+/* Used to allocate new swap item for new memory allocation and cow page for write. */ -+struct mali_page_node *_mali_mem_swap_page_node_allocate(void) -+{ -+ struct mali_page_node *m_page; -+ -+ m_page = _mali_page_node_allocate(MALI_PAGE_NODE_SWAP); -+ -+ if (NULL == m_page) { -+ return NULL; -+ } -+ -+ m_page->swap_it = mali_mem_swap_alloc_swap_item(); -+ -+ if (NULL == m_page->swap_it) { -+ kfree(m_page); -+ return NULL; -+ } -+ -+ return m_page; -+} -+ -+_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page) -+{ -+ -+ mali_mem_swap_free_swap_item(m_page->swap_it); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _mali_mem_swap_page_node_free(struct mali_page_node *m_page) -+{ -+ _mali_mem_swap_put_page_node(m_page); -+ -+ kfree(m_page); -+ -+ return; -+} -+ -+u32 mali_mem_swap_free(mali_mem_swap *swap_mem) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ u32 free_pages_nr = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(swap_mem); -+ -+ list_for_each_entry_safe(m_page, m_tmp, &swap_mem->pages, list) { -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); -+ -+ /* free the page node and release the swap item, if the ref count is 1, -+ * then need also free the swap item. */ -+ list_del(&m_page->list); -+ if (1 == _mali_page_node_get_ref_count(m_page)) { -+ free_pages_nr++; -+ } -+ -+ _mali_mem_swap_page_node_free(m_page); -+ } -+ -+ return free_pages_nr; -+} -+ -+static u32 mali_mem_swap_cow_free(mali_mem_cow *cow_mem) -+{ -+ struct mali_page_node *m_page, *m_tmp; -+ u32 free_pages_nr = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(cow_mem); -+ -+ list_for_each_entry_safe(m_page, m_tmp, &cow_mem->pages, list) { -+ MALI_DEBUG_ASSERT(m_page->type == MALI_PAGE_NODE_SWAP); -+ -+ /* free the page node and release the swap item, if the ref count is 1, -+ * then need also free the swap item. */ -+ list_del(&m_page->list); -+ if (1 == _mali_page_node_get_ref_count(m_page)) { -+ free_pages_nr++; -+ } -+ -+ _mali_mem_swap_page_node_free(m_page); -+ } -+ -+ return free_pages_nr; -+} -+ -+u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped) -+{ -+ mali_mem_allocation *alloc; -+ u32 free_pages_nr = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem_bkend); -+ alloc = mem_bkend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ if (is_mali_mapped) { -+ mali_mem_swap_mali_unmap(alloc); -+ } -+ -+ mali_memory_swap_list_backend_delete(mem_bkend); -+ -+ mutex_lock(&mem_bkend->mutex); -+ /* To make sure the given memory backend was unlocked from Mali side, -+ * and then free this memory block. */ -+ mali_mem_swap_unlock_single_mem_backend(mem_bkend); -+ mutex_unlock(&mem_bkend->mutex); -+ -+ if (MALI_MEM_SWAP == mem_bkend->type) { -+ free_pages_nr = mali_mem_swap_free(&mem_bkend->swap_mem); -+ } else { -+ free_pages_nr = mali_mem_swap_cow_free(&mem_bkend->cow_mem); -+ } -+ -+ return free_pages_nr; -+} -+ -+mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node) -+{ -+ MALI_DEBUG_ASSERT(NULL != page_node); -+ -+ page_node->swap_it->page = shmem_read_mapping_page(global_swap_space, page_node->swap_it->idx); -+ -+ if (IS_ERR(page_node->swap_it->page)) { -+ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: failed to swap in page with index: %d.\n", page_node->swap_it->idx)); -+ return MALI_FALSE; -+ } -+ -+ /* Ensure page is flushed from CPU caches. */ -+ page_node->swap_it->dma_addr = dma_map_page(&mali_platform_device->dev, page_node->swap_it->page, -+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); -+ -+ return MALI_TRUE; -+} -+ -+int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx) -+{ -+ size_t page_count = PAGE_ALIGN(size) / PAGE_SIZE; -+ struct mali_page_node *m_page; -+ long system_free_size; -+ u32 i, index; -+ mali_bool ret; -+ -+ MALI_DEBUG_ASSERT(NULL != swap_mem); -+ MALI_DEBUG_ASSERT(NULL != bkend_idx); -+ MALI_DEBUG_ASSERT(page_count <= MALI_SWAP_GLOBAL_SWAP_FILE_INDEX_RESERVE); -+ -+ if (atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE + size > mali_mem_os_allocator.allocation_limit) { -+ MALI_DEBUG_PRINT(2, ("Mali Mem: Unable to allocate %u bytes. Currently allocated: %lu, max limit %lu\n", -+ size, -+ atomic_read(&mali_mem_os_allocator.allocated_pages) * _MALI_OSK_MALI_PAGE_SIZE, -+ mali_mem_os_allocator.allocation_limit)); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ INIT_LIST_HEAD(&swap_mem->pages); -+ swap_mem->count = page_count; -+ index = mali_mem_swap_idx_range_alloc(page_count); -+ -+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == index) { -+ MALI_PRINT_ERROR(("Mali Swap: Failed to allocate continuous index for swappable Mali memory.")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ for (i = 0; i < page_count; i++) { -+ m_page = _mali_mem_swap_page_node_allocate(); -+ -+ if (NULL == m_page) { -+ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Failed to allocate mali page node.")); -+ swap_mem->count = i; -+ -+ mali_mem_swap_free(swap_mem); -+ mali_mem_swap_idx_range_free(index + i, page_count - i); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ m_page->swap_it->idx = index + i; -+ -+ ret = mali_mem_swap_in_page_node(m_page); -+ -+ if (MALI_FALSE == ret) { -+ MALI_DEBUG_PRINT_ERROR(("SWAP Mem: Allocate new page from SHMEM file failed.")); -+ _mali_mem_swap_page_node_free(m_page); -+ mali_mem_swap_idx_range_free(index + i + 1, page_count - i - 1); -+ -+ swap_mem->count = i; -+ mali_mem_swap_free(swap_mem); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ list_add_tail(&m_page->list, &swap_mem->pages); -+ } -+ -+ system_free_size = global_zone_page_state(NR_FREE_PAGES) * PAGE_SIZE; -+ -+ if ((system_free_size < mali_mem_swap_out_threshold_value) -+ && (mem_backend_swapped_pool_size > (mali_mem_swap_out_threshold_value >> 2)) -+ && mali_utilization_enabled()) { -+ _mali_osk_wq_schedule_work(mali_mem_swap_out_workq); -+ } -+ -+ *bkend_idx = index; -+ return 0; -+} -+ -+void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc) -+{ -+ struct mali_session_data *session; -+ -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ mali_session_memory_lock(session); -+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ mali_session_memory_unlock(session); -+} -+ -+ -+/* Insert these pages from shmem to mali page table*/ -+_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props) -+{ -+ struct mali_page_directory *pagedir = session->page_directory; -+ struct mali_page_node *m_page; -+ dma_addr_t phys; -+ u32 virt = vaddr; -+ u32 prop = props; -+ -+ list_for_each_entry(m_page, &swap_mem->pages, list) { -+ MALI_DEBUG_ASSERT(NULL != m_page->swap_it->page); -+ phys = m_page->swap_it->dma_addr; -+ -+ mali_mmu_pagedir_update(pagedir, virt, phys, MALI_MMU_PAGE_SIZE, prop); -+ virt += MALI_MMU_PAGE_SIZE; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+int mali_mem_swap_in_pages(struct mali_pp_job *job) -+{ -+ u32 num_memory_cookies; -+ struct mali_session_data *session; -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_allocation *mali_alloc = NULL; -+ mali_mem_backend *mem_bkend = NULL; -+ struct mali_page_node *m_page; -+ mali_bool swap_in_success = MALI_TRUE; -+ int i; -+ -+ MALI_DEBUG_ASSERT_POINTER(job); -+ -+ num_memory_cookies = mali_pp_job_num_memory_cookies(job); -+ session = mali_pp_job_get_session(job); -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ for (i = 0; i < num_memory_cookies; i++) { -+ -+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); -+ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); -+ if (NULL == mali_vma_node) { -+ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; -+ swap_in_success = MALI_FALSE; -+ MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); -+ continue; -+ } -+ -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ MALI_DEBUG_ASSERT(NULL != mali_alloc); -+ -+ if (MALI_MEM_SWAP != mali_alloc->type && -+ MALI_MEM_COW != mali_alloc->type) { -+ continue; -+ } -+ -+ /* Get backend memory & Map on GPU */ -+ mutex_lock(&mali_idr_mutex); -+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(NULL != mem_bkend); -+ -+ /* We neednot hold backend's lock here, race safe.*/ -+ if ((MALI_MEM_COW == mem_bkend->type) && -+ (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { -+ continue; -+ } -+ -+ mutex_lock(&mem_bkend->mutex); -+ -+ /* When swap_in_success is MALI_FALSE, it means this job has memory backend that could not be swapped in, -+ * and it will be aborted in mali scheduler, so here, we just mark those memory cookies which -+ * should not be swapped out when delete job to invalide */ -+ if (MALI_FALSE == swap_in_success) { -+ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; -+ mutex_unlock(&mem_bkend->mutex); -+ continue; -+ } -+ -+ /* Before swap in, checking if this memory backend has been swapped in by the latest flushed jobs. */ -+ ++mem_bkend->using_count; -+ -+ if (1 < mem_bkend->using_count) { -+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); -+ mutex_unlock(&mem_bkend->mutex); -+ continue; -+ } -+ -+ if (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN != (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)) { -+ mutex_unlock(&mem_bkend->mutex); -+ continue; -+ } -+ -+ -+ list_for_each_entry(m_page, &mem_bkend->swap_mem.pages, list) { -+ if (MALI_FALSE == mali_mem_swap_in_page_node(m_page)) { -+ /* Don't have enough memory to swap in page, so release pages have already been swapped -+ * in and then mark this pp job to be fail. */ -+ mali_mem_swap_unlock_partial_locked_mem_backend(mem_bkend, m_page); -+ swap_in_success = MALI_FALSE; -+ break; -+ } -+ } -+ -+ if (swap_in_success) { -+#ifdef MALI_MEM_SWAP_TRACKING -+ mem_backend_swapped_unlock_size -= mem_bkend->size; -+#endif -+ _mali_osk_mutex_wait(session->memory_lock); -+ mali_mem_swap_mali_map(&mem_bkend->swap_mem, session, mali_alloc->mali_mapping.addr, mali_alloc->mali_mapping.properties); -+ _mali_osk_mutex_signal(session->memory_lock); -+ -+ /* Remove the unlock flag from mem backend flags, mark this backend has been swapped in. */ -+ mem_bkend->flags &= ~(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN); -+ mutex_unlock(&mem_bkend->mutex); -+ } else { -+ --mem_bkend->using_count; -+ /* Marking that this backend is not swapped in, need not to be processed anymore. */ -+ job->memory_cookies[i] = MALI_SWAP_INVALIDATE_MALI_ADDRESS; -+ mutex_unlock(&mem_bkend->mutex); -+ } -+ } -+ -+ job->swap_status = swap_in_success ? MALI_SWAP_IN_SUCC : MALI_SWAP_IN_FAIL; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+int mali_mem_swap_out_pages(struct mali_pp_job *job) -+{ -+ u32 num_memory_cookies; -+ struct mali_session_data *session; -+ struct mali_vma_node *mali_vma_node = NULL; -+ mali_mem_allocation *mali_alloc = NULL; -+ mali_mem_backend *mem_bkend = NULL; -+ int i; -+ -+ MALI_DEBUG_ASSERT_POINTER(job); -+ -+ num_memory_cookies = mali_pp_job_num_memory_cookies(job); -+ session = mali_pp_job_get_session(job); -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ -+ for (i = 0; i < num_memory_cookies; i++) { -+ u32 mali_addr = mali_pp_job_get_memory_cookie(job, i); -+ -+ if (MALI_SWAP_INVALIDATE_MALI_ADDRESS == mali_addr) { -+ continue; -+ } -+ -+ mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, mali_addr, 0); -+ -+ if (NULL == mali_vma_node) { -+ MALI_PRINT_ERROR(("SWAP Mem: failed to find mali_vma_node through Mali address: 0x%08x.\n", mali_addr)); -+ continue; -+ } -+ -+ mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node); -+ MALI_DEBUG_ASSERT(NULL != mali_alloc); -+ -+ if (MALI_MEM_SWAP != mali_alloc->type && -+ MALI_MEM_COW != mali_alloc->type) { -+ continue; -+ } -+ -+ mutex_lock(&mali_idr_mutex); -+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(NULL != mem_bkend); -+ -+ /* We neednot hold backend's lock here, race safe.*/ -+ if ((MALI_MEM_COW == mem_bkend->type) && -+ (!(mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED))) { -+ continue; -+ } -+ -+ mutex_lock(&mem_bkend->mutex); -+ -+ MALI_DEBUG_ASSERT(0 < mem_bkend->using_count); -+ -+ /* Reducing the using_count of mem backend means less pp job are using this memory backend, -+ * if this count get to zero, it means no pp job is using it now, could put it to swap out list. */ -+ --mem_bkend->using_count; -+ -+ if (0 < mem_bkend->using_count) { -+ mutex_unlock(&mem_bkend->mutex); -+ continue; -+ } -+ mutex_unlock(&mem_bkend->mutex); -+ -+ mali_memory_swap_list_backend_add(mem_bkend); -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) -+{ -+ struct mali_page_node *m_page, *found_node = NULL; -+ struct page *found_page; -+ mali_mem_swap *swap = NULL; -+ mali_mem_cow *cow = NULL; -+ dma_addr_t dma_addr; -+ u32 i = 0; -+ -+ if (MALI_MEM_SWAP == mem_bkend->type) { -+ swap = &mem_bkend->swap_mem; -+ list_for_each_entry(m_page, &swap->pages, list) { -+ if (i == offset) { -+ found_node = m_page; -+ break; -+ } -+ i++; -+ } -+ } else { -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); -+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (MALI_MEM_BACKEND_FLAG_SWAP_COWED & mem_bkend->flags)); -+ -+ cow = &mem_bkend->cow_mem; -+ list_for_each_entry(m_page, &cow->pages, list) { -+ if (i == offset) { -+ found_node = m_page; -+ break; -+ } -+ i++; -+ } -+ } -+ -+ if (NULL == found_node) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ found_page = shmem_read_mapping_page(global_swap_space, found_node->swap_it->idx); -+ -+ if (!IS_ERR(found_page)) { -+ lock_page(found_page); -+ dma_addr = dma_map_page(&mali_platform_device->dev, found_page, -+ 0, _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); -+ dma_unmap_page(&mali_platform_device->dev, dma_addr, -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); -+ -+ *pagep = found_page; -+ } else { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) -+{ -+ struct mali_page_node *m_page, *found_node = NULL, *new_node = NULL; -+ mali_mem_cow *cow = NULL; -+ u32 i = 0; -+ -+ MALI_DEBUG_ASSERT(MALI_MEM_COW == mem_bkend->type); -+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_SWAP_COWED == (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED)); -+ MALI_DEBUG_ASSERT(MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN == (MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN & mem_bkend->flags)); -+ MALI_DEBUG_ASSERT(!mali_memory_swap_backend_in_swapped_pool(mem_bkend)); -+ -+ cow = &mem_bkend->cow_mem; -+ list_for_each_entry(m_page, &cow->pages, list) { -+ if (i == offset) { -+ found_node = m_page; -+ break; -+ } -+ i++; -+ } -+ -+ if (NULL == found_node) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ new_node = _mali_mem_swap_page_node_allocate(); -+ -+ if (NULL == new_node) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ new_node->swap_it->idx = mali_mem_swap_idx_alloc(); -+ -+ if (_MALI_OSK_BITMAP_INVALIDATE_INDEX == new_node->swap_it->idx) { -+ MALI_DEBUG_PRINT(1, ("Failed to allocate swap index in swap CoW on demand.\n")); -+ kfree(new_node->swap_it); -+ kfree(new_node); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ if (MALI_FALSE == mali_mem_swap_in_page_node(new_node)) { -+ _mali_mem_swap_page_node_free(new_node); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ /* swap in found node for copy in kernel. */ -+ if (MALI_FALSE == mali_mem_swap_in_page_node(found_node)) { -+ mali_mem_swap_out_page_node(new_node); -+ _mali_mem_swap_page_node_free(new_node); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ _mali_mem_cow_copy_page(found_node, new_node); -+ -+ list_replace(&found_node->list, &new_node->list); -+ -+ if (1 != _mali_page_node_get_ref_count(found_node)) { -+ atomic_add(1, &mem_bkend->mali_allocation->session->mali_mem_allocated_pages); -+ if (atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE > mem_bkend->mali_allocation->session->max_mali_mem_allocated_size) { -+ mem_bkend->mali_allocation->session->max_mali_mem_allocated_size = atomic_read(&mem_bkend->mali_allocation->session->mali_mem_allocated_pages) * MALI_MMU_PAGE_SIZE; -+ } -+ mem_bkend->cow_mem.change_pages_nr++; -+ } -+ -+ mali_mem_swap_out_page_node(found_node); -+ _mali_mem_swap_page_node_free(found_node); -+ -+ /* When swap in the new page node, we have called dma_map_page for this page.\n */ -+ dma_unmap_page(&mali_platform_device->dev, new_node->swap_it->dma_addr, -+ _MALI_OSK_MALI_PAGE_SIZE, DMA_TO_DEVICE); -+ -+ lock_page(new_node->swap_it->page); -+ -+ *pagep = new_node->swap_it->page; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+#ifdef MALI_MEM_SWAP_TRACKING -+void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size) -+{ -+ *swap_pool_size = mem_backend_swapped_pool_size; -+ *unlock_size = mem_backend_swapped_unlock_size; -+} -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h -new file mode 100755 -index 000000000..5810960e2 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_swap_alloc.h -@@ -0,0 +1,121 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_SWAP_ALLOC_H__ -+#define __MALI_MEMORY_SWAP_ALLOC_H__ -+ -+#include "mali_osk.h" -+#include "mali_session.h" -+ -+#include "mali_memory_types.h" -+#include "mali_pp_job.h" -+ -+/** -+ * Initialize memory swapping module. -+ */ -+_mali_osk_errcode_t mali_mem_swap_init(void); -+ -+void mali_mem_swap_term(void); -+ -+/** -+ * Return global share memory file to other modules. -+ */ -+struct file *mali_mem_swap_get_global_swap_file(void); -+ -+/** -+ * Unlock the given memory backend and pages in it could be swapped out by kernel. -+ */ -+void mali_mem_swap_unlock_single_mem_backend(mali_mem_backend *mem_bkend); -+ -+/** -+ * Remove the given memory backend from global swap list. -+ */ -+void mali_memory_swap_list_backend_delete(mali_mem_backend *mem_bkend); -+ -+/** -+ * Add the given memory backend to global swap list. -+ */ -+void mali_memory_swap_list_backend_add(mali_mem_backend *mem_bkend); -+ -+/** -+ * Allocate 1 index from bitmap used as page index in global swap file. -+ */ -+u32 mali_mem_swap_idx_alloc(void); -+ -+void mali_mem_swap_idx_free(u32 idx); -+ -+/** -+ * Allocate a new swap item without page index. -+ */ -+struct mali_swap_item *mali_mem_swap_alloc_swap_item(void); -+ -+/** -+ * Free a swap item, truncate the corresponding space in page cache and free index of page. -+ */ -+void mali_mem_swap_free_swap_item(mali_swap_item *swap_item); -+ -+/** -+ * Allocate a page node with swap item. -+ */ -+struct mali_page_node *_mali_mem_swap_page_node_allocate(void); -+ -+/** -+ * Reduce the reference count of given page node and if return 0, just free this page node. -+ */ -+_mali_osk_errcode_t _mali_mem_swap_put_page_node(struct mali_page_node *m_page); -+ -+void _mali_mem_swap_page_node_free(struct mali_page_node *m_page); -+ -+/** -+ * Free a swappable memory backend. -+ */ -+u32 mali_mem_swap_free(mali_mem_swap *swap_mem); -+ -+/** -+ * Ummap and free. -+ */ -+u32 mali_mem_swap_release(mali_mem_backend *mem_bkend, mali_bool is_mali_mapped); -+ -+/** -+ * Read in a page from global swap file with the pre-allcated page index. -+ */ -+mali_bool mali_mem_swap_in_page_node(struct mali_page_node *page_node); -+ -+int mali_mem_swap_alloc_pages(mali_mem_swap *swap_mem, u32 size, u32 *bkend_idx); -+ -+_mali_osk_errcode_t mali_mem_swap_mali_map(mali_mem_swap *swap_mem, struct mali_session_data *session, u32 vaddr, u32 props); -+ -+void mali_mem_swap_mali_unmap(mali_mem_allocation *alloc); -+ -+/** -+ * When pp job created, we need swap in all of memory backend needed by this pp job. -+ */ -+int mali_mem_swap_in_pages(struct mali_pp_job *job); -+ -+/** -+ * Put all of memory backends used this pp job to the global swap list. -+ */ -+int mali_mem_swap_out_pages(struct mali_pp_job *job); -+ -+/** -+ * This will be called in page fault to process CPU read&write. -+ */ -+int mali_mem_swap_allocate_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep) ; -+ -+/** -+ * Used to process cow on demand for swappable memory backend. -+ */ -+int mali_mem_swap_cow_page_on_demand(mali_mem_backend *mem_bkend, u32 offset, struct page **pagep); -+ -+#ifdef MALI_MEM_SWAP_TRACKING -+void mali_mem_swap_tracking(u32 *swap_pool_size, u32 *unlock_size); -+#endif -+#endif /* __MALI_MEMORY_SWAP_ALLOC_H__ */ -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h -new file mode 100755 -index 000000000..33db40929 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_types.h -@@ -0,0 +1,219 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_TYPES_H__ -+#define __MALI_MEMORY_TYPES_H__ -+ -+#include -+ -+#if defined(CONFIG_MALI400_UMP) -+#include "ump_kernel_interface.h" -+#endif -+ -+typedef u32 mali_address_t; -+ -+typedef enum mali_mem_type { -+ MALI_MEM_OS, -+ MALI_MEM_EXTERNAL, -+ MALI_MEM_SWAP, -+ MALI_MEM_DMA_BUF, -+ MALI_MEM_UMP, -+ MALI_MEM_BLOCK, -+ MALI_MEM_COW, -+ MALI_MEM_SECURE, -+ MALI_MEM_TYPE_MAX, -+} mali_mem_type; -+ -+typedef struct mali_block_item { -+ /* for block type, the block_phy is alway page size align -+ * so use low 12bit used for ref_cout. -+ */ -+ unsigned long phy_addr; -+} mali_block_item; -+ -+/** -+ * idx is used to locate the given page in the address space of swap file. -+ * ref_count is used to mark how many memory backends are using this item. -+ */ -+typedef struct mali_swap_item { -+ u32 idx; -+ atomic_t ref_count; -+ struct page *page; -+ dma_addr_t dma_addr; -+} mali_swap_item; -+ -+typedef enum mali_page_node_type { -+ MALI_PAGE_NODE_OS, -+ MALI_PAGE_NODE_BLOCK, -+ MALI_PAGE_NODE_SWAP, -+} mali_page_node_type; -+ -+typedef struct mali_page_node { -+ struct list_head list; -+ union { -+ struct page *page; -+ mali_block_item *blk_it; /*pointer to block item*/ -+ mali_swap_item *swap_it; -+ }; -+ -+ u32 type; -+} mali_page_node; -+ -+typedef struct mali_mem_os_mem { -+ struct list_head pages; -+ u32 count; -+} mali_mem_os_mem; -+ -+typedef struct mali_mem_dma_buf { -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ struct mali_dma_buf_attachment *attachment; -+#endif -+} mali_mem_dma_buf; -+ -+typedef struct mali_mem_external { -+ dma_addr_t phys; -+ u32 size; -+} mali_mem_external; -+ -+typedef struct mali_mem_ump { -+#if defined(CONFIG_MALI400_UMP) -+ ump_dd_handle handle; -+#endif -+} mali_mem_ump; -+ -+typedef struct block_allocator_allocation { -+ /* The list will be released in reverse order */ -+ struct block_info *last_allocated; -+ u32 mapping_length; -+ struct block_allocator *info; -+} block_allocator_allocation; -+ -+typedef struct mali_mem_block_mem { -+ struct list_head pfns; -+ u32 count; -+} mali_mem_block_mem; -+ -+typedef struct mali_mem_virt_mali_mapping { -+ mali_address_t addr; /* Virtual Mali address */ -+ u32 properties; /* MMU Permissions + cache, must match MMU HW */ -+} mali_mem_virt_mali_mapping; -+ -+typedef struct mali_mem_virt_cpu_mapping { -+ void __user *addr; -+ struct vm_area_struct *vma; -+} mali_mem_virt_cpu_mapping; -+ -+#define MALI_MEM_ALLOCATION_VALID_MAGIC 0xdeda110c -+#define MALI_MEM_ALLOCATION_FREED_MAGIC 0x10101010 -+ -+typedef struct mali_mm_node { -+ /* MALI GPU vaddr start, use u32 for mmu only support 32bit address*/ -+ uint32_t start; /* GPU vaddr */ -+ uint32_t size; /* GPU allocation virtual size */ -+ unsigned allocated : 1; -+} mali_mm_node; -+ -+typedef struct mali_vma_node { -+ struct mali_mm_node vm_node; -+ struct rb_node vm_rb; -+} mali_vma_node; -+ -+ -+typedef struct mali_mem_allocation { -+ MALI_DEBUG_CODE(u32 magic); -+ mali_mem_type type; /**< Type of memory */ -+ u32 flags; /**< Flags for this allocation */ -+ -+ struct mali_session_data *session; /**< Pointer to session that owns the allocation */ -+ -+ mali_mem_virt_cpu_mapping cpu_mapping; /**< CPU mapping */ -+ mali_mem_virt_mali_mapping mali_mapping; /**< Mali mapping */ -+ -+ /* add for new memory system */ -+ struct mali_vma_node mali_vma_node; -+ u32 vsize; /* virtual size*/ -+ u32 psize; /* physical backend memory size*/ -+ struct list_head list; -+ s32 backend_handle; /* idr for mem_backend */ -+ _mali_osk_atomic_t mem_alloc_refcount; -+} mali_mem_allocation; -+ -+struct mali_mem_os_allocator { -+ spinlock_t pool_lock; -+ struct list_head pool_pages; -+ size_t pool_count; -+ -+ atomic_t allocated_pages; -+ size_t allocation_limit; -+ -+ struct shrinker shrinker; -+ struct delayed_work timed_shrinker; -+ struct workqueue_struct *wq; -+}; -+ -+/* COW backend memory type */ -+typedef struct mali_mem_cow { -+ struct list_head pages; /**< all pages for this cow backend allocation, -+ including new allocated pages for modified range*/ -+ u32 count; /**< number of pages */ -+ s32 change_pages_nr; -+} mali_mem_cow; -+ -+typedef struct mali_mem_swap { -+ struct list_head pages; -+ u32 count; -+} mali_mem_swap; -+ -+typedef struct mali_mem_secure { -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ struct dma_buf *buf; -+ struct dma_buf_attachment *attachment; -+ struct sg_table *sgt; -+#endif -+ u32 count; -+} mali_mem_secure; -+ -+#define MALI_MEM_BACKEND_FLAG_COWED (0x1) /* COW has happen on this backend */ -+#define MALI_MEM_BACKEND_FLAG_COW_CPU_NO_WRITE (0x2) /* This is an COW backend, mapped as not allowed cpu to write */ -+#define MALI_MEM_BACKEND_FLAG_SWAP_COWED (0x4) /* Mark the given backend is cowed from swappable memory. */ -+/* Mark this backend is not swapped_in in MALI driver, and before using it, -+ * we should swap it in and set up corresponding page table. */ -+#define MALI_MEM_BACKEND_FLAG_UNSWAPPED_IN (0x8) -+#define MALI_MEM_BACKEND_FLAG_NOT_BINDED (0x1 << 5) /* this backend it not back with physical memory, used for defer bind */ -+#define MALI_MEM_BACKEND_FLAG_BINDED (0x1 << 6) /* this backend it back with physical memory, used for defer bind */ -+ -+typedef struct mali_mem_backend { -+ mali_mem_type type; /**< Type of backend memory */ -+ u32 flags; /**< Flags for this allocation */ -+ u32 size; -+ /* Union selected by type. */ -+ union { -+ mali_mem_os_mem os_mem; /**< MALI_MEM_OS */ -+ mali_mem_external ext_mem; /**< MALI_MEM_EXTERNAL */ -+ mali_mem_dma_buf dma_buf; /**< MALI_MEM_DMA_BUF */ -+ mali_mem_ump ump_mem; /**< MALI_MEM_UMP */ -+ mali_mem_block_mem block_mem; /**< MALI_MEM_BLOCK */ -+ mali_mem_cow cow_mem; -+ mali_mem_swap swap_mem; -+ mali_mem_secure secure_mem; -+ }; -+ mali_mem_allocation *mali_allocation; -+ struct mutex mutex; -+ mali_mem_type cow_type; -+ -+ struct list_head list; /**< Used to link swappable memory backend to the global swappable list */ -+ int using_count; /**< Mark how many PP jobs are using this memory backend */ -+ u32 start_idx; /**< If the correspondign vma of this backend is linear, this value will be used to set vma->vm_pgoff */ -+} mali_mem_backend; -+ -+#define MALI_MEM_FLAG_MALI_GUARD_PAGE (_MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) -+#define MALI_MEM_FLAG_DONT_CPU_MAP (1 << 1) -+#define MALI_MEM_FLAG_CAN_RESIZE (_MALI_MEMORY_ALLOCATE_RESIZEABLE) -+#endif /* __MALI_MEMORY_TYPES__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c -new file mode 100755 -index 000000000..666d4b0fb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.c -@@ -0,0 +1,154 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_kernel_linux.h" -+#include "mali_memory.h" -+#include "ump_kernel_interface.h" -+ -+static int mali_mem_ump_map(mali_mem_backend *mem_backend) -+{ -+ ump_dd_handle ump_mem; -+ mali_mem_allocation *alloc; -+ struct mali_session_data *session; -+ u32 nr_blocks; -+ u32 i; -+ ump_dd_physical_block *ump_blocks; -+ struct mali_page_directory *pagedir; -+ u32 offset = 0; -+ _mali_osk_errcode_t err; -+ -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); -+ -+ alloc = mem_backend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ ump_mem = mem_backend->ump_mem.handle; -+ MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); -+ -+ nr_blocks = ump_dd_phys_block_count_get(ump_mem); -+ if (nr_blocks == 0) { -+ MALI_DEBUG_PRINT(1, ("No block count\n")); -+ return -EINVAL; -+ } -+ -+ ump_blocks = _mali_osk_malloc(sizeof(*ump_blocks) * nr_blocks); -+ if (NULL == ump_blocks) { -+ return -ENOMEM; -+ } -+ -+ if (UMP_DD_INVALID == ump_dd_phys_blocks_get(ump_mem, ump_blocks, nr_blocks)) { -+ _mali_osk_free(ump_blocks); -+ return -EFAULT; -+ } -+ -+ pagedir = session->page_directory; -+ -+ mali_session_memory_lock(session); -+ -+ err = mali_mem_mali_map_prepare(alloc); -+ if (_MALI_OSK_ERR_OK != err) { -+ MALI_DEBUG_PRINT(1, ("Mapping of UMP memory failed\n")); -+ -+ _mali_osk_free(ump_blocks); -+ mali_session_memory_unlock(session); -+ return -ENOMEM; -+ } -+ -+ for (i = 0; i < nr_blocks; ++i) { -+ u32 virt = alloc->mali_vma_node.vm_node.start + offset; -+ -+ MALI_DEBUG_PRINT(7, ("Mapping in 0x%08x size %d\n", ump_blocks[i].addr , ump_blocks[i].size)); -+ -+ mali_mmu_pagedir_update(pagedir, virt, ump_blocks[i].addr, -+ ump_blocks[i].size, MALI_MMU_FLAGS_DEFAULT); -+ -+ offset += ump_blocks[i].size; -+ } -+ -+ if (alloc->flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { -+ u32 virt = alloc->mali_vma_node.vm_node.start + offset; -+ -+ /* Map in an extra virtual guard page at the end of the VMA */ -+ MALI_DEBUG_PRINT(6, ("Mapping in extra guard page\n")); -+ -+ mali_mmu_pagedir_update(pagedir, virt, ump_blocks[0].addr, _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT); -+ -+ offset += _MALI_OSK_MALI_PAGE_SIZE; -+ } -+ mali_session_memory_unlock(session); -+ _mali_osk_free(ump_blocks); -+ return 0; -+} -+ -+static void mali_mem_ump_unmap(mali_mem_allocation *alloc) -+{ -+ struct mali_session_data *session; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ session = alloc->session; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ mali_session_memory_lock(session); -+ mali_mem_mali_map_free(session, alloc->psize, alloc->mali_vma_node.vm_node.start, -+ alloc->flags); -+ mali_session_memory_unlock(session); -+} -+ -+int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags) -+{ -+ ump_dd_handle ump_mem; -+ int ret; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); -+ -+ MALI_DEBUG_PRINT(3, -+ ("Requested to map ump memory with secure id %d into virtual memory 0x%08X, size 0x%08X\n", -+ secure_id, alloc->mali_vma_node.vm_node.start, alloc->mali_vma_node.vm_node.size)); -+ -+ ump_mem = ump_dd_handle_create_from_secure_id(secure_id); -+ if (UMP_DD_HANDLE_INVALID == ump_mem) MALI_ERROR(_MALI_OSK_ERR_FAULT); -+ alloc->flags |= MALI_MEM_FLAG_DONT_CPU_MAP; -+ if (flags & _MALI_MAP_EXTERNAL_MAP_GUARD_PAGE) { -+ alloc->flags |= MALI_MEM_FLAG_MALI_GUARD_PAGE; -+ } -+ -+ mem_backend->ump_mem.handle = ump_mem; -+ -+ ret = mali_mem_ump_map(mem_backend); -+ if (0 != ret) { -+ ump_dd_reference_release(ump_mem); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ MALI_DEBUG_PRINT(3, ("Returning from UMP bind\n")); -+ return _MALI_OSK_ERR_OK; -+} -+ -+void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend) -+{ -+ ump_dd_handle ump_mem; -+ mali_mem_allocation *alloc; -+ MALI_DEBUG_ASSERT_POINTER(mem_backend); -+ MALI_DEBUG_ASSERT(MALI_MEM_UMP == mem_backend->type); -+ ump_mem = mem_backend->ump_mem.handle; -+ MALI_DEBUG_ASSERT(UMP_DD_HANDLE_INVALID != ump_mem); -+ -+ alloc = mem_backend->mali_allocation; -+ MALI_DEBUG_ASSERT_POINTER(alloc); -+ mali_mem_ump_unmap(alloc); -+ ump_dd_reference_release(ump_mem); -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h -new file mode 100755 -index 000000000..c314c8dcb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_ump.h -@@ -0,0 +1,29 @@ -+/* -+ * Copyright (C) 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_UMP_BUF_H__ -+#define __MALI_MEMORY_UMP_BUF_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include "mali_uk_types.h" -+#include "mali_osk.h" -+#include "mali_memory.h" -+ -+int mali_mem_bind_ump_buf(mali_mem_allocation *alloc, mali_mem_backend *mem_backend, u32 secure_id, u32 flags); -+void mali_mem_unbind_ump_buf(mali_mem_backend *mem_backend); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_MEMORY_DMA_BUF_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c -new file mode 100755 -index 000000000..8e13e923c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.c -@@ -0,0 +1,158 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_osk_mali.h" -+#include "mali_kernel_linux.h" -+#include "mali_scheduler.h" -+ -+#include "mali_memory.h" -+#include "mali_memory_os_alloc.h" -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+#include "mali_memory_dma_buf.h" -+#include "mali_memory_secure.h" -+#endif -+#if defined(CONFIG_MALI400_UMP) -+#include "mali_memory_ump.h" -+#endif -+#include "mali_memory_external.h" -+#include "mali_memory_manager.h" -+#include "mali_memory_virtual.h" -+#include "mali_memory_cow.h" -+#include "mali_memory_block_alloc.h" -+#include "mali_memory_swap_alloc.h" -+ -+ -+ -+/** -+*function @_mali_free_allocation_mem - free a memory allocation -+*/ -+static u32 _mali_free_allocation_mem(mali_mem_allocation *mali_alloc) -+{ -+ mali_mem_backend *mem_bkend = NULL; -+ u32 free_pages_nr = 0; -+ -+ struct mali_session_data *session = mali_alloc->session; -+ MALI_DEBUG_PRINT(4, (" _mali_free_allocation_mem, psize =0x%x! \n", mali_alloc->psize)); -+ if (0 == mali_alloc->psize) -+ goto out; -+ -+ /* Get backend memory & Map on CPU */ -+ mutex_lock(&mali_idr_mutex); -+ mem_bkend = idr_find(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ MALI_DEBUG_ASSERT(NULL != mem_bkend); -+ -+ switch (mem_bkend->type) { -+ case MALI_MEM_OS: -+ free_pages_nr = mali_mem_os_release(mem_bkend); -+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); -+ break; -+ case MALI_MEM_UMP: -+#if defined(CONFIG_MALI400_UMP) -+ mali_mem_unbind_ump_buf(mem_bkend); -+ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); -+#else -+ MALI_DEBUG_PRINT(1, ("UMP not supported\n")); -+#endif -+ break; -+ case MALI_MEM_DMA_BUF: -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ mali_mem_unbind_dma_buf(mem_bkend); -+ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); -+#else -+ MALI_DEBUG_PRINT(1, ("DMA not supported\n")); -+#endif -+ break; -+ case MALI_MEM_EXTERNAL: -+ mali_mem_unbind_ext_buf(mem_bkend); -+ atomic_sub(mem_bkend->size / MALI_MMU_PAGE_SIZE, &session->mali_mem_array[mem_bkend->type]); -+ break; -+ -+ case MALI_MEM_BLOCK: -+ free_pages_nr = mali_mem_block_release(mem_bkend); -+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); -+ break; -+ -+ case MALI_MEM_COW: -+ if (mem_bkend->flags & MALI_MEM_BACKEND_FLAG_SWAP_COWED) { -+ free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); -+ } else { -+ free_pages_nr = mali_mem_cow_release(mem_bkend, MALI_TRUE); -+ } -+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); -+ break; -+ case MALI_MEM_SWAP: -+ free_pages_nr = mali_mem_swap_release(mem_bkend, MALI_TRUE); -+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); -+ atomic_sub(free_pages_nr, &session->mali_mem_array[mem_bkend->type]); -+ break; -+ case MALI_MEM_SECURE: -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ free_pages_nr = mali_mem_secure_release(mem_bkend); -+ atomic_sub(free_pages_nr, &session->mali_mem_allocated_pages); -+#else -+ MALI_DEBUG_PRINT(1, ("DMA not supported for mali secure memory\n")); -+#endif -+ break; -+ default: -+ MALI_DEBUG_PRINT(1, ("mem type %d is not in the mali_mem_type enum.\n", mem_bkend->type)); -+ break; -+ } -+ -+ /*Remove backend memory idex */ -+ mutex_lock(&mali_idr_mutex); -+ idr_remove(&mali_backend_idr, mali_alloc->backend_handle); -+ mutex_unlock(&mali_idr_mutex); -+ kfree(mem_bkend); -+out: -+ /* remove memory allocation */ -+ mali_vma_offset_remove(&session->allocation_mgr, &mali_alloc->mali_vma_node); -+ mali_mem_allocation_struct_destory(mali_alloc); -+ return free_pages_nr; -+} -+ -+/** -+* ref_count for allocation -+*/ -+u32 mali_allocation_unref(struct mali_mem_allocation **alloc) -+{ -+ u32 free_pages_nr = 0; -+ mali_mem_allocation *mali_alloc = *alloc; -+ *alloc = NULL; -+ if (0 == _mali_osk_atomic_dec_return(&mali_alloc->mem_alloc_refcount)) { -+ free_pages_nr = _mali_free_allocation_mem(mali_alloc); -+ } -+ return free_pages_nr; -+} -+ -+void mali_allocation_ref(struct mali_mem_allocation *alloc) -+{ -+ _mali_osk_atomic_inc(&alloc->mem_alloc_refcount); -+} -+ -+void mali_free_session_allocations(struct mali_session_data *session) -+{ -+ struct mali_mem_allocation *entry, *next; -+ -+ MALI_DEBUG_PRINT(4, (" mali_free_session_allocations! \n")); -+ -+ list_for_each_entry_safe(entry, next, &session->allocation_mgr.head, list) { -+ mali_allocation_unref(&entry); -+ } -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h -new file mode 100755 -index 000000000..33ac99509 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_util.h -@@ -0,0 +1,20 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_MEMORY_UTIL_H__ -+#define __MALI_MEMORY_UTIL_H__ -+ -+u32 mali_allocation_unref(struct mali_mem_allocation **alloc); -+ -+void mali_allocation_ref(struct mali_mem_allocation *alloc); -+ -+void mali_free_session_allocations(struct mali_session_data *session); -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c -new file mode 100755 -index 000000000..0b31e3a23 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.c -@@ -0,0 +1,127 @@ -+/* -+ * Copyright (C) 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_osk_mali.h" -+#include "mali_kernel_linux.h" -+#include "mali_scheduler.h" -+#include "mali_memory_os_alloc.h" -+#include "mali_memory_manager.h" -+#include "mali_memory_virtual.h" -+ -+ -+/** -+*internal helper to link node into the rb-tree -+*/ -+static inline void _mali_vma_offset_add_rb(struct mali_allocation_manager *mgr, -+ struct mali_vma_node *node) -+{ -+ struct rb_node **iter = &mgr->allocation_mgr_rb.rb_node; -+ struct rb_node *parent = NULL; -+ struct mali_vma_node *iter_node; -+ -+ while (likely(*iter)) { -+ parent = *iter; -+ iter_node = rb_entry(*iter, struct mali_vma_node, vm_rb); -+ -+ if (node->vm_node.start < iter_node->vm_node.start) -+ iter = &(*iter)->rb_left; -+ else if (node->vm_node.start > iter_node->vm_node.start) -+ iter = &(*iter)->rb_right; -+ else -+ MALI_DEBUG_ASSERT(0); -+ } -+ -+ rb_link_node(&node->vm_rb, parent, iter); -+ rb_insert_color(&node->vm_rb, &mgr->allocation_mgr_rb); -+} -+ -+/** -+ * mali_vma_offset_add() - Add offset node to RB Tree -+ */ -+int mali_vma_offset_add(struct mali_allocation_manager *mgr, -+ struct mali_vma_node *node) -+{ -+ int ret = 0; -+ write_lock(&mgr->vm_lock); -+ -+ if (node->vm_node.allocated) { -+ goto out; -+ } -+ -+ _mali_vma_offset_add_rb(mgr, node); -+ /* set to allocated */ -+ node->vm_node.allocated = 1; -+ -+out: -+ write_unlock(&mgr->vm_lock); -+ return ret; -+} -+ -+/** -+ * mali_vma_offset_remove() - Remove offset node from RB tree -+ */ -+void mali_vma_offset_remove(struct mali_allocation_manager *mgr, -+ struct mali_vma_node *node) -+{ -+ write_lock(&mgr->vm_lock); -+ -+ if (node->vm_node.allocated) { -+ rb_erase(&node->vm_rb, &mgr->allocation_mgr_rb); -+ memset(&node->vm_node, 0, sizeof(node->vm_node)); -+ } -+ write_unlock(&mgr->vm_lock); -+} -+ -+/** -+* mali_vma_offset_search - Search the node in RB tree -+*/ -+struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, -+ unsigned long start, unsigned long pages) -+{ -+ struct mali_vma_node *node, *best; -+ struct rb_node *iter; -+ unsigned long offset; -+ read_lock(&mgr->vm_lock); -+ -+ iter = mgr->allocation_mgr_rb.rb_node; -+ best = NULL; -+ -+ while (likely(iter)) { -+ node = rb_entry(iter, struct mali_vma_node, vm_rb); -+ offset = node->vm_node.start; -+ if (start >= offset) { -+ iter = iter->rb_right; -+ best = node; -+ if (start == offset) -+ break; -+ } else { -+ iter = iter->rb_left; -+ } -+ } -+ -+ if (best) { -+ offset = best->vm_node.start + best->vm_node.size; -+ if (offset <= start + pages) -+ best = NULL; -+ } -+ read_unlock(&mgr->vm_lock); -+ -+ return best; -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h -new file mode 100755 -index 000000000..fd03ed9f2 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_memory_virtual.h -@@ -0,0 +1,35 @@ -+/* -+ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#ifndef __MALI_GPU_VMEM_H__ -+#define __MALI_GPU_VMEM_H__ -+ -+#include "mali_osk.h" -+#include "mali_session.h" -+#include -+#include -+#include -+#include -+#include -+#include "mali_memory_types.h" -+#include "mali_memory_os_alloc.h" -+#include "mali_memory_manager.h" -+ -+ -+ -+int mali_vma_offset_add(struct mali_allocation_manager *mgr, -+ struct mali_vma_node *node); -+ -+void mali_vma_offset_remove(struct mali_allocation_manager *mgr, -+ struct mali_vma_node *node); -+ -+struct mali_vma_node *mali_vma_offset_search(struct mali_allocation_manager *mgr, -+ unsigned long start, unsigned long pages); -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c -new file mode 100755 -index 000000000..5bc0e52eb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_atomics.c -@@ -0,0 +1,59 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_atomics.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include "mali_osk.h" -+#include -+#include "mali_kernel_common.h" -+ -+void _mali_osk_atomic_dec(_mali_osk_atomic_t *atom) -+{ -+ atomic_dec((atomic_t *)&atom->u.val); -+} -+ -+u32 _mali_osk_atomic_dec_return(_mali_osk_atomic_t *atom) -+{ -+ return atomic_dec_return((atomic_t *)&atom->u.val); -+} -+ -+void _mali_osk_atomic_inc(_mali_osk_atomic_t *atom) -+{ -+ atomic_inc((atomic_t *)&atom->u.val); -+} -+ -+u32 _mali_osk_atomic_inc_return(_mali_osk_atomic_t *atom) -+{ -+ return atomic_inc_return((atomic_t *)&atom->u.val); -+} -+ -+void _mali_osk_atomic_init(_mali_osk_atomic_t *atom, u32 val) -+{ -+ MALI_DEBUG_ASSERT_POINTER(atom); -+ atomic_set((atomic_t *)&atom->u.val, val); -+} -+ -+u32 _mali_osk_atomic_read(_mali_osk_atomic_t *atom) -+{ -+ return atomic_read((atomic_t *)&atom->u.val); -+} -+ -+void _mali_osk_atomic_term(_mali_osk_atomic_t *atom) -+{ -+ MALI_IGNORE(atom); -+} -+ -+u32 _mali_osk_atomic_xchg(_mali_osk_atomic_t *atom, u32 val) -+{ -+ return atomic_xchg((atomic_t *)&atom->u.val, val); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c -new file mode 100755 -index 000000000..fb9ccd2ad ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_bitmap.c -@@ -0,0 +1,152 @@ -+/* -+ * Copyright (C) 2010, 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_bitmap.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "common/mali_kernel_common.h" -+#include "mali_osk_types.h" -+#include "mali_osk.h" -+ -+u32 _mali_osk_bitmap_alloc(struct _mali_osk_bitmap *bitmap) -+{ -+ u32 obj; -+ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ -+ _mali_osk_spinlock_lock(bitmap->lock); -+ -+ obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->reserve); -+ -+ if (obj < bitmap->max) { -+ set_bit(obj, bitmap->table); -+ } else { -+ obj = -1; -+ } -+ -+ if (obj != -1) -+ --bitmap->avail; -+ _mali_osk_spinlock_unlock(bitmap->lock); -+ -+ return obj; -+} -+ -+void _mali_osk_bitmap_free(struct _mali_osk_bitmap *bitmap, u32 obj) -+{ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ -+ _mali_osk_bitmap_free_range(bitmap, obj, 1); -+} -+ -+u32 _mali_osk_bitmap_alloc_range(struct _mali_osk_bitmap *bitmap, int cnt) -+{ -+ u32 obj; -+ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ -+ if (0 >= cnt) { -+ return -1; -+ } -+ -+ if (1 == cnt) { -+ return _mali_osk_bitmap_alloc(bitmap); -+ } -+ -+ _mali_osk_spinlock_lock(bitmap->lock); -+ obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, -+ bitmap->last, cnt, 0); -+ -+ if (obj >= bitmap->max) { -+ obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max, -+ bitmap->reserve, cnt, 0); -+ } -+ -+ if (obj < bitmap->max) { -+ bitmap_set(bitmap->table, obj, cnt); -+ -+ bitmap->last = (obj + cnt); -+ if (bitmap->last >= bitmap->max) { -+ bitmap->last = bitmap->reserve; -+ } -+ } else { -+ obj = -1; -+ } -+ -+ if (obj != -1) { -+ bitmap->avail -= cnt; -+ } -+ -+ _mali_osk_spinlock_unlock(bitmap->lock); -+ -+ return obj; -+} -+ -+u32 _mali_osk_bitmap_avail(struct _mali_osk_bitmap *bitmap) -+{ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ -+ return bitmap->avail; -+} -+ -+void _mali_osk_bitmap_free_range(struct _mali_osk_bitmap *bitmap, u32 obj, int cnt) -+{ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ -+ _mali_osk_spinlock_lock(bitmap->lock); -+ bitmap_clear(bitmap->table, obj, cnt); -+ bitmap->last = min(bitmap->last, obj); -+ -+ bitmap->avail += cnt; -+ _mali_osk_spinlock_unlock(bitmap->lock); -+} -+ -+int _mali_osk_bitmap_init(struct _mali_osk_bitmap *bitmap, u32 num, u32 reserve) -+{ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ MALI_DEBUG_ASSERT(reserve <= num); -+ -+ bitmap->reserve = reserve; -+ bitmap->last = reserve; -+ bitmap->max = num; -+ bitmap->avail = num - reserve; -+ bitmap->lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_UNORDERED, _MALI_OSK_LOCK_ORDER_FIRST); -+ if (!bitmap->lock) { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * -+ sizeof(long), GFP_KERNEL); -+ if (!bitmap->table) { -+ _mali_osk_spinlock_term(bitmap->lock); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _mali_osk_bitmap_term(struct _mali_osk_bitmap *bitmap) -+{ -+ MALI_DEBUG_ASSERT_POINTER(bitmap); -+ -+ if (NULL != bitmap->lock) { -+ _mali_osk_spinlock_term(bitmap->lock); -+ } -+ -+ if (NULL != bitmap->table) { -+ kfree(bitmap->table); -+ } -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c -new file mode 100755 -index 000000000..5c8b9ceab ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_irq.c -@@ -0,0 +1,200 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_irq.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include /* For memory allocation */ -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+ -+typedef struct _mali_osk_irq_t_struct { -+ u32 irqnum; -+ void *data; -+ _mali_osk_irq_uhandler_t uhandler; -+} mali_osk_irq_object_t; -+ -+typedef irqreturn_t (*irq_handler_func_t)(int, void *, struct pt_regs *); -+static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id); /* , struct pt_regs *regs*/ -+ -+#if defined(DEBUG) -+ -+struct test_interrupt_data { -+ _mali_osk_irq_ack_t ack_func; -+ void *probe_data; -+ mali_bool interrupt_received; -+ wait_queue_head_t wq; -+}; -+ -+static irqreturn_t test_interrupt_upper_half(int port_name, void *dev_id) -+{ -+ irqreturn_t ret = IRQ_NONE; -+ struct test_interrupt_data *data = (struct test_interrupt_data *)dev_id; -+ -+ if (_MALI_OSK_ERR_OK == data->ack_func(data->probe_data)) { -+ data->interrupt_received = MALI_TRUE; -+ wake_up(&data->wq); -+ ret = IRQ_HANDLED; -+ } -+ -+ return ret; -+} -+ -+static _mali_osk_errcode_t test_interrupt(u32 irqnum, -+ _mali_osk_irq_trigger_t trigger_func, -+ _mali_osk_irq_ack_t ack_func, -+ void *probe_data, -+ const char *description) -+{ -+ unsigned long irq_flags = 0; -+ struct test_interrupt_data data = { -+ .ack_func = ack_func, -+ .probe_data = probe_data, -+ .interrupt_received = MALI_FALSE, -+ }; -+ -+#if defined(CONFIG_MALI_SHARED_INTERRUPTS) -+ irq_flags |= IRQF_SHARED; -+#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ -+ -+ if (0 != request_irq(irqnum, test_interrupt_upper_half, irq_flags, description, &data)) { -+ MALI_DEBUG_PRINT(2, ("Unable to install test IRQ handler for core '%s'\n", description)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ init_waitqueue_head(&data.wq); -+ -+ trigger_func(probe_data); -+ wait_event_timeout(data.wq, data.interrupt_received, 100); -+ -+ free_irq(irqnum, &data); -+ -+ if (data.interrupt_received) { -+ MALI_DEBUG_PRINT(3, ("%s: Interrupt test OK\n", description)); -+ return _MALI_OSK_ERR_OK; -+ } else { -+ MALI_PRINT_ERROR(("%s: Failed interrupt test on %u\n", description, irqnum)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+} -+ -+#endif /* defined(DEBUG) */ -+ -+_mali_osk_irq_t *_mali_osk_irq_init(u32 irqnum, _mali_osk_irq_uhandler_t uhandler, void *int_data, _mali_osk_irq_trigger_t trigger_func, _mali_osk_irq_ack_t ack_func, void *probe_data, const char *description) -+{ -+ mali_osk_irq_object_t *irq_object; -+ unsigned long irq_flags = 0; -+ -+#if defined(CONFIG_MALI_SHARED_INTERRUPTS) -+ irq_flags |= IRQF_SHARED; -+#endif /* defined(CONFIG_MALI_SHARED_INTERRUPTS) */ -+ -+ irq_object = kmalloc(sizeof(mali_osk_irq_object_t), GFP_KERNEL); -+ if (NULL == irq_object) { -+ return NULL; -+ } -+ -+ if (-1 == irqnum) { -+ /* Probe for IRQ */ -+ if ((NULL != trigger_func) && (NULL != ack_func)) { -+ unsigned long probe_count = 3; -+ _mali_osk_errcode_t err; -+ int irq; -+ -+ MALI_DEBUG_PRINT(2, ("Probing for irq\n")); -+ -+ do { -+ unsigned long mask; -+ -+ mask = probe_irq_on(); -+ trigger_func(probe_data); -+ -+ _mali_osk_time_ubusydelay(5); -+ -+ irq = probe_irq_off(mask); -+ err = ack_func(probe_data); -+ } while (irq < 0 && (err == _MALI_OSK_ERR_OK) && probe_count--); -+ -+ if (irq < 0 || (_MALI_OSK_ERR_OK != err)) irqnum = -1; -+ else irqnum = irq; -+ } else irqnum = -1; /* no probe functions, fault */ -+ -+ if (-1 != irqnum) { -+ /* found an irq */ -+ MALI_DEBUG_PRINT(2, ("Found irq %d\n", irqnum)); -+ } else { -+ MALI_DEBUG_PRINT(2, ("Probe for irq failed\n")); -+ } -+ } -+ -+ irq_object->irqnum = irqnum; -+ irq_object->uhandler = uhandler; -+ irq_object->data = int_data; -+ -+ if (-1 == irqnum) { -+ MALI_DEBUG_PRINT(2, ("No IRQ for core '%s' found during probe\n", description)); -+ kfree(irq_object); -+ return NULL; -+ } -+ -+#if defined(DEBUG) -+ /* Verify that the configured interrupt settings are working */ -+ if (_MALI_OSK_ERR_OK != test_interrupt(irqnum, trigger_func, ack_func, probe_data, description)) { -+ MALI_DEBUG_PRINT(2, ("Test of IRQ(%d) handler for core '%s' failed\n", irqnum, description)); -+ kfree(irq_object); -+ return NULL; -+ } -+#endif -+ -+ if (0 != request_irq(irqnum, irq_handler_upper_half, irq_flags, description, irq_object)) { -+ MALI_DEBUG_PRINT(2, ("Unable to install IRQ handler for core '%s'\n", description)); -+ kfree(irq_object); -+ return NULL; -+ } -+ -+ return irq_object; -+} -+ -+void _mali_osk_irq_term(_mali_osk_irq_t *irq) -+{ -+ mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)irq; -+ free_irq(irq_object->irqnum, irq_object); -+ kfree(irq_object); -+} -+ -+ -+/** This function is called directly in interrupt context from the OS just after -+ * the CPU get the hw-irq from mali, or other devices on the same IRQ-channel. -+ * It is registered one of these function for each mali core. When an interrupt -+ * arrives this function will be called equal times as registered mali cores. -+ * That means that we only check one mali core in one function call, and the -+ * core we check for each turn is given by the \a dev_id variable. -+ * If we detect an pending interrupt on the given core, we mask the interrupt -+ * out by settging the core's IRQ_MASK register to zero. -+ * Then we schedule the mali_core_irq_handler_bottom_half to run as high priority -+ * work queue job. -+ */ -+static irqreturn_t irq_handler_upper_half(int port_name, void *dev_id) /* , struct pt_regs *regs*/ -+{ -+ irqreturn_t ret = IRQ_NONE; -+ mali_osk_irq_object_t *irq_object = (mali_osk_irq_object_t *)dev_id; -+ -+ if (_MALI_OSK_ERR_OK == irq_object->uhandler(irq_object->data)) { -+ ret = IRQ_HANDLED; -+ } -+ -+ return ret; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c -new file mode 100755 -index 000000000..ed5f0b0da ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.c -@@ -0,0 +1,287 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_locks.c -+ * Implemenation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include "mali_osk_locks.h" -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+ -+ -+#ifdef DEBUG -+#ifdef LOCK_ORDER_CHECKING -+static DEFINE_SPINLOCK(lock_tracking_lock); -+static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid); -+static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid); -+static const char *const lock_order_to_string(_mali_osk_lock_order_t order); -+#endif /* LOCK_ORDER_CHECKING */ -+ -+void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) -+{ -+ checker->orig_flags = flags; -+ checker->owner = 0; -+ -+#ifdef LOCK_ORDER_CHECKING -+ checker->order = order; -+ checker->next = NULL; -+#endif -+} -+ -+void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker) -+{ -+ checker->owner = _mali_osk_get_tid(); -+ -+#ifdef LOCK_ORDER_CHECKING -+ if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { -+ if (!add_lock_to_log_and_check(checker, _mali_osk_get_tid())) { -+ printk(KERN_ERR "%d: ERROR lock %p taken while holding a lock of a higher order.\n", -+ _mali_osk_get_tid(), checker); -+ dump_stack(); -+ } -+ } -+#endif -+} -+ -+void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker) -+{ -+ -+#ifdef LOCK_ORDER_CHECKING -+ if (!(checker->orig_flags & _MALI_OSK_LOCKFLAG_UNORDERED)) { -+ remove_lock_from_log(checker, _mali_osk_get_tid()); -+ } -+#endif -+ checker->owner = 0; -+} -+ -+ -+#ifdef LOCK_ORDER_CHECKING -+/* Lock order checking -+ * ------------------- -+ * -+ * To assure that lock ordering scheme defined by _mali_osk_lock_order_t is strictly adhered to, the -+ * following function will, together with a linked list and some extra members in _mali_osk_lock_debug_s, -+ * make sure that a lock that is taken has a higher order than the current highest-order lock a -+ * thread holds. -+ * -+ * This is done in the following manner: -+ * - A linked list keeps track of locks held by a thread. -+ * - A `next' pointer is added to each lock. This is used to chain the locks together. -+ * - When taking a lock, the `add_lock_to_log_and_check' makes sure that taking -+ * the given lock is legal. It will follow the linked list to find the last -+ * lock taken by this thread. If the last lock's order was lower than the -+ * lock that is to be taken, it appends the new lock to the list and returns -+ * true, if not, it return false. This return value is assert()'ed on in -+ * _mali_osk_lock_wait(). -+ */ -+ -+static struct _mali_osk_lock_debug_s *lock_lookup_list; -+ -+static void dump_lock_tracking_list(void) -+{ -+ struct _mali_osk_lock_debug_s *l; -+ u32 n = 1; -+ -+ /* print list for debugging purposes */ -+ l = lock_lookup_list; -+ -+ while (NULL != l) { -+ printk(" [lock: %p, tid_owner: %d, order: %d] ->", l, l->owner, l->order); -+ l = l->next; -+ MALI_DEBUG_ASSERT(n++ < 100); -+ } -+ printk(" NULL\n"); -+} -+ -+static int tracking_list_length(void) -+{ -+ struct _mali_osk_lock_debug_s *l; -+ u32 n = 0; -+ l = lock_lookup_list; -+ -+ while (NULL != l) { -+ l = l->next; -+ n++; -+ MALI_DEBUG_ASSERT(n < 100); -+ } -+ return n; -+} -+ -+static mali_bool add_lock_to_log_and_check(struct _mali_osk_lock_debug_s *lock, uint32_t tid) -+{ -+ mali_bool ret = MALI_FALSE; -+ _mali_osk_lock_order_t highest_order_for_tid = _MALI_OSK_LOCK_ORDER_FIRST; -+ struct _mali_osk_lock_debug_s *highest_order_lock = (struct _mali_osk_lock_debug_s *)0xbeefbabe; -+ struct _mali_osk_lock_debug_s *l; -+ unsigned long local_lock_flag; -+ u32 len; -+ -+ spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); -+ len = tracking_list_length(); -+ -+ l = lock_lookup_list; -+ if (NULL == l) { /* This is the first lock taken by this thread -- record and return true */ -+ lock_lookup_list = lock; -+ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); -+ return MALI_TRUE; -+ } else { -+ /* Traverse the locks taken and find the lock of the highest order. -+ * Since several threads may hold locks, each lock's owner must be -+ * checked so that locks not owned by this thread can be ignored. */ -+ for (;;) { -+ MALI_DEBUG_ASSERT_POINTER(l); -+ if (tid == l->owner && l->order >= highest_order_for_tid) { -+ highest_order_for_tid = l->order; -+ highest_order_lock = l; -+ } -+ -+ if (NULL != l->next) { -+ l = l->next; -+ } else { -+ break; -+ } -+ } -+ -+ l->next = lock; -+ l->next = NULL; -+ } -+ -+ /* We have now found the highest order lock currently held by this thread and can see if it is -+ * legal to take the requested lock. */ -+ ret = highest_order_for_tid < lock->order; -+ -+ if (!ret) { -+ printk(KERN_ERR "Took lock of order %d (%s) while holding lock of order %d (%s)\n", -+ lock->order, lock_order_to_string(lock->order), -+ highest_order_for_tid, lock_order_to_string(highest_order_for_tid)); -+ dump_lock_tracking_list(); -+ } -+ -+ if (len + 1 != tracking_list_length()) { -+ printk(KERN_ERR "************ lock: %p\n", lock); -+ printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); -+ dump_lock_tracking_list(); -+ MALI_DEBUG_ASSERT_POINTER(NULL); -+ } -+ -+ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); -+ return ret; -+} -+ -+static void remove_lock_from_log(struct _mali_osk_lock_debug_s *lock, uint32_t tid) -+{ -+ struct _mali_osk_lock_debug_s *curr; -+ struct _mali_osk_lock_debug_s *prev = NULL; -+ unsigned long local_lock_flag; -+ u32 len; -+ u32 n = 0; -+ -+ spin_lock_irqsave(&lock_tracking_lock, local_lock_flag); -+ len = tracking_list_length(); -+ curr = lock_lookup_list; -+ -+ if (NULL == curr) { -+ printk(KERN_ERR "Error: Lock tracking list was empty on call to remove_lock_from_log\n"); -+ dump_lock_tracking_list(); -+ } -+ -+ MALI_DEBUG_ASSERT_POINTER(curr); -+ -+ -+ while (lock != curr) { -+ prev = curr; -+ -+ MALI_DEBUG_ASSERT_POINTER(curr); -+ curr = curr->next; -+ MALI_DEBUG_ASSERT(n++ < 100); -+ } -+ -+ if (NULL == prev) { -+ lock_lookup_list = curr->next; -+ } else { -+ MALI_DEBUG_ASSERT_POINTER(curr); -+ MALI_DEBUG_ASSERT_POINTER(prev); -+ prev->next = curr->next; -+ } -+ -+ lock->next = NULL; -+ -+ if (len - 1 != tracking_list_length()) { -+ printk(KERN_ERR "************ lock: %p\n", lock); -+ printk(KERN_ERR "************ before: %d *** after: %d ****\n", len, tracking_list_length()); -+ dump_lock_tracking_list(); -+ MALI_DEBUG_ASSERT_POINTER(NULL); -+ } -+ -+ spin_unlock_irqrestore(&lock_tracking_lock, local_lock_flag); -+} -+ -+static const char *const lock_order_to_string(_mali_osk_lock_order_t order) -+{ -+ switch (order) { -+ case _MALI_OSK_LOCK_ORDER_SESSIONS: -+ return "_MALI_OSK_LOCK_ORDER_SESSIONS"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_MEM_SESSION: -+ return "_MALI_OSK_LOCK_ORDER_MEM_SESSION"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_MEM_INFO: -+ return "_MALI_OSK_LOCK_ORDER_MEM_INFO"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_MEM_PT_CACHE: -+ return "_MALI_OSK_LOCK_ORDER_MEM_PT_CACHE"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP: -+ return "_MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_PM_EXECUTION: -+ return "_MALI_OSK_LOCK_ORDER_PM_EXECUTION"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_EXECUTOR: -+ return "_MALI_OSK_LOCK_ORDER_EXECUTOR"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM: -+ return "_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_SCHEDULER: -+ return "_MALI_OSK_LOCK_ORDER_SCHEDULER"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED: -+ return "_MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_DMA_COMMAND: -+ return "_MALI_OSK_LOCK_ORDER_DMA_COMMAND"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_PROFILING: -+ return "_MALI_OSK_LOCK_ORDER_PROFILING"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_L2: -+ return "_MALI_OSK_LOCK_ORDER_L2"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_L2_COMMAND: -+ return "_MALI_OSK_LOCK_ORDER_L2_COMMAND"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_UTILIZATION: -+ return "_MALI_OSK_LOCK_ORDER_UTILIZATION"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS: -+ return "_MALI_OSK_LOCK_ORDER_SESSION_PENDING_JOBS"; -+ break; -+ case _MALI_OSK_LOCK_ORDER_PM_STATE: -+ return "_MALI_OSK_LOCK_ORDER_PM_STATE"; -+ break; -+ default: -+ return ""; -+ } -+} -+#endif /* LOCK_ORDER_CHECKING */ -+#endif /* DEBUG */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h -new file mode 100755 -index 000000000..6fd5af952 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_locks.h -@@ -0,0 +1,326 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_locks.h -+ * Defines OS abstraction of lock and mutex -+ */ -+#ifndef _MALI_OSK_LOCKS_H -+#define _MALI_OSK_LOCKS_H -+ -+#include -+#include -+#include -+ -+#include -+ -+#include "mali_osk_types.h" -+ -+#ifdef _cplusplus -+extern "C" { -+#endif -+ -+ /* When DEBUG is enabled, this struct will be used to track owner, mode and order checking */ -+#ifdef DEBUG -+ struct _mali_osk_lock_debug_s { -+ u32 owner; -+ _mali_osk_lock_flags_t orig_flags; -+ _mali_osk_lock_order_t order; -+ struct _mali_osk_lock_debug_s *next; -+ }; -+#endif -+ -+ /* Anstraction of spinlock_t */ -+ struct _mali_osk_spinlock_s { -+#ifdef DEBUG -+ struct _mali_osk_lock_debug_s checker; -+#endif -+ spinlock_t spinlock; -+ }; -+ -+ /* Abstration of spinlock_t and lock flag which is used to store register's state before locking */ -+ struct _mali_osk_spinlock_irq_s { -+#ifdef DEBUG -+ struct _mali_osk_lock_debug_s checker; -+#endif -+ -+ spinlock_t spinlock; -+ unsigned long flags; -+ }; -+ -+ /* Abstraction of rw_semaphore in OS */ -+ struct _mali_osk_mutex_rw_s { -+#ifdef DEBUG -+ struct _mali_osk_lock_debug_s checker; -+ _mali_osk_lock_mode_t mode; -+#endif -+ -+ struct rw_semaphore rw_sema; -+ }; -+ -+ /* Mutex and mutex_interruptible functions share the same osk mutex struct */ -+ struct _mali_osk_mutex_s { -+#ifdef DEBUG -+ struct _mali_osk_lock_debug_s checker; -+#endif -+ struct mutex mutex; -+ }; -+ -+#ifdef DEBUG -+ /** @brief _mali_osk_locks_debug_init/add/remove() functions are declared when DEBUG is enabled and -+ * defined in file mali_osk_locks.c. When LOCK_ORDER_CHECKING is enabled, calling these functions when we -+ * init/lock/unlock a lock/mutex, we could track lock order of a given tid. */ -+ void _mali_osk_locks_debug_init(struct _mali_osk_lock_debug_s *checker, _mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order); -+ void _mali_osk_locks_debug_add(struct _mali_osk_lock_debug_s *checker); -+ void _mali_osk_locks_debug_remove(struct _mali_osk_lock_debug_s *checker); -+ -+ /** @brief This function can return a given lock's owner when DEBUG is enabled. */ -+ static inline u32 _mali_osk_lock_get_owner(struct _mali_osk_lock_debug_s *lock) -+ { -+ return lock->owner; -+ } -+#else -+#define _mali_osk_locks_debug_init(x, y, z) do {} while (0) -+#define _mali_osk_locks_debug_add(x) do {} while (0) -+#define _mali_osk_locks_debug_remove(x) do {} while (0) -+#endif -+ -+ /** @brief Before use _mali_osk_spin_lock, init function should be used to allocate memory and initial spinlock*/ -+ static inline _mali_osk_spinlock_t *_mali_osk_spinlock_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) -+ { -+ _mali_osk_spinlock_t *lock = NULL; -+ -+ lock = kmalloc(sizeof(_mali_osk_spinlock_t), GFP_KERNEL); -+ if (NULL == lock) { -+ return NULL; -+ } -+ spin_lock_init(&lock->spinlock); -+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); -+ return lock; -+ } -+ -+ /** @brief Lock a spinlock */ -+ static inline void _mali_osk_spinlock_lock(_mali_osk_spinlock_t *lock) -+ { -+ BUG_ON(NULL == lock); -+ spin_lock(&lock->spinlock); -+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); -+ } -+ -+ /** @brief Unlock a spinlock */ -+ static inline void _mali_osk_spinlock_unlock(_mali_osk_spinlock_t *lock) -+ { -+ BUG_ON(NULL == lock); -+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); -+ spin_unlock(&lock->spinlock); -+ } -+ -+ /** @brief Free a memory block which the argument lock pointed to and its type must be -+ * _mali_osk_spinlock_t *. */ -+ static inline void _mali_osk_spinlock_term(_mali_osk_spinlock_t *lock) -+ { -+ /* Parameter validation */ -+ BUG_ON(NULL == lock); -+ -+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ -+ kfree(lock); -+ } -+ -+ /** @brief Before _mali_osk_spinlock_irq_lock/unlock/term() is called, init function should be -+ * called to initial spinlock and flags in struct _mali_osk_spinlock_irq_t. */ -+ static inline _mali_osk_spinlock_irq_t *_mali_osk_spinlock_irq_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) -+ { -+ _mali_osk_spinlock_irq_t *lock = NULL; -+ lock = kmalloc(sizeof(_mali_osk_spinlock_irq_t), GFP_KERNEL); -+ -+ if (NULL == lock) { -+ return NULL; -+ } -+ -+ lock->flags = 0; -+ spin_lock_init(&lock->spinlock); -+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); -+ return lock; -+ } -+ -+ /** @brief Lock spinlock and save the register's state */ -+ static inline void _mali_osk_spinlock_irq_lock(_mali_osk_spinlock_irq_t *lock) -+ { -+ unsigned long tmp_flags; -+ -+ BUG_ON(NULL == lock); -+ spin_lock_irqsave(&lock->spinlock, tmp_flags); -+ lock->flags = tmp_flags; -+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); -+ } -+ -+ /** @brief Unlock spinlock with saved register's state */ -+ static inline void _mali_osk_spinlock_irq_unlock(_mali_osk_spinlock_irq_t *lock) -+ { -+ BUG_ON(NULL == lock); -+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); -+ spin_unlock_irqrestore(&lock->spinlock, lock->flags); -+ } -+ -+ /** @brief Destroy a given memory block which lock pointed to, and the lock type must be -+ * _mali_osk_spinlock_irq_t *. */ -+ static inline void _mali_osk_spinlock_irq_term(_mali_osk_spinlock_irq_t *lock) -+ { -+ /* Parameter validation */ -+ BUG_ON(NULL == lock); -+ -+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ -+ kfree(lock); -+ } -+ -+ /** @brief Before _mali_osk_mutex_rw_wait/signal/term() is called, we should call -+ * _mali_osk_mutex_rw_init() to kmalloc a memory block and initial part of elements in it. */ -+ static inline _mali_osk_mutex_rw_t *_mali_osk_mutex_rw_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) -+ { -+ _mali_osk_mutex_rw_t *lock = NULL; -+ -+ lock = kmalloc(sizeof(_mali_osk_mutex_rw_t), GFP_KERNEL); -+ -+ if (NULL == lock) { -+ return NULL; -+ } -+ -+ init_rwsem(&lock->rw_sema); -+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); -+ return lock; -+ } -+ -+ /** @brief When call _mali_osk_mutex_rw_wait/signal() functions, the second argument mode -+ * should be assigned with value _MALI_OSK_LOCKMODE_RO or _MALI_OSK_LOCKMODE_RW */ -+ static inline void _mali_osk_mutex_rw_wait(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) -+ { -+ BUG_ON(NULL == lock); -+ BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); -+ -+ if (mode == _MALI_OSK_LOCKMODE_RO) { -+ down_read(&lock->rw_sema); -+ } else { -+ down_write(&lock->rw_sema); -+ } -+ -+#ifdef DEBUG -+ if (mode == _MALI_OSK_LOCKMODE_RW) { -+ lock->mode = mode; -+ } else { /* mode == _MALI_OSK_LOCKMODE_RO */ -+ lock->mode = mode; -+ } -+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); -+#endif -+ } -+ -+ /** @brief Up lock->rw_sema with up_read/write() accordinf argument mode's value. */ -+ static inline void _mali_osk_mutex_rw_signal(_mali_osk_mutex_rw_t *lock, _mali_osk_lock_mode_t mode) -+ { -+ BUG_ON(NULL == lock); -+ BUG_ON(!(_MALI_OSK_LOCKMODE_RO == mode || _MALI_OSK_LOCKMODE_RW == mode)); -+#ifdef DEBUG -+ /* make sure the thread releasing the lock actually was the owner */ -+ if (mode == _MALI_OSK_LOCKMODE_RW) { -+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); -+ /* This lock now has no owner */ -+ lock->checker.owner = 0; -+ } -+#endif -+ -+ if (mode == _MALI_OSK_LOCKMODE_RO) { -+ up_read(&lock->rw_sema); -+ } else { -+ up_write(&lock->rw_sema); -+ } -+ } -+ -+ /** @brief Free a given memory block which lock pointed to and its type must be -+ * _mali_sok_mutex_rw_t *. */ -+ static inline void _mali_osk_mutex_rw_term(_mali_osk_mutex_rw_t *lock) -+ { -+ /* Parameter validation */ -+ BUG_ON(NULL == lock); -+ -+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ -+ kfree(lock); -+ } -+ -+ /** @brief Mutex & mutex_interruptible share the same init and term function, because they have the -+ * same osk mutex struct, and the difference between them is which locking function they use */ -+ static inline _mali_osk_mutex_t *_mali_osk_mutex_init(_mali_osk_lock_flags_t flags, _mali_osk_lock_order_t order) -+ { -+ _mali_osk_mutex_t *lock = NULL; -+ -+ lock = kmalloc(sizeof(_mali_osk_mutex_t), GFP_KERNEL); -+ -+ if (NULL == lock) { -+ return NULL; -+ } -+ mutex_init(&lock->mutex); -+ -+ _mali_osk_locks_debug_init((struct _mali_osk_lock_debug_s *)lock, flags, order); -+ return lock; -+ } -+ -+ /** @brief Lock the lock->mutex with mutex_lock_interruptible function */ -+ static inline _mali_osk_errcode_t _mali_osk_mutex_wait_interruptible(_mali_osk_mutex_t *lock) -+ { -+ _mali_osk_errcode_t err = _MALI_OSK_ERR_OK; -+ -+ BUG_ON(NULL == lock); -+ -+ if (mutex_lock_interruptible(&lock->mutex)) { -+ printk(KERN_WARNING "Mali: Can not lock mutex\n"); -+ err = _MALI_OSK_ERR_RESTARTSYSCALL; -+ } -+ -+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); -+ return err; -+ } -+ -+ /** @brief Unlock the lock->mutex which is locked with mutex_lock_interruptible() function. */ -+ static inline void _mali_osk_mutex_signal_interruptible(_mali_osk_mutex_t *lock) -+ { -+ BUG_ON(NULL == lock); -+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); -+ mutex_unlock(&lock->mutex); -+ } -+ -+ /** @brief Lock the lock->mutex just with mutex_lock() function which could not be interruptted. */ -+ static inline void _mali_osk_mutex_wait(_mali_osk_mutex_t *lock) -+ { -+ BUG_ON(NULL == lock); -+ mutex_lock(&lock->mutex); -+ _mali_osk_locks_debug_add((struct _mali_osk_lock_debug_s *)lock); -+ } -+ -+ /** @brief Unlock the lock->mutex which is locked with mutex_lock() function. */ -+ static inline void _mali_osk_mutex_signal(_mali_osk_mutex_t *lock) -+ { -+ BUG_ON(NULL == lock); -+ _mali_osk_locks_debug_remove((struct _mali_osk_lock_debug_s *)lock); -+ mutex_unlock(&lock->mutex); -+ } -+ -+ /** @brief Free a given memory block which lock point. */ -+ static inline void _mali_osk_mutex_term(_mali_osk_mutex_t *lock) -+ { -+ /* Parameter validation */ -+ BUG_ON(NULL == lock); -+ -+ /* Linux requires no explicit termination of spinlocks, semaphores, or rw_semaphores */ -+ kfree(lock); -+ } -+ -+#ifdef _cplusplus -+} -+#endif -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c -new file mode 100755 -index 000000000..994b04dad ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_low_level_mem.c -@@ -0,0 +1,146 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_low_level_mem.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include -+#include -+#include -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_ukk.h" -+ -+void _mali_osk_mem_barrier(void) -+{ -+ mb(); -+} -+ -+void _mali_osk_write_mem_barrier(void) -+{ -+ wmb(); -+} -+ -+mali_io_address _mali_osk_mem_mapioregion(uintptr_t phys, u32 size, const char *description) -+{ -+ return (mali_io_address)ioremap(phys, size); -+} -+ -+void _mali_osk_mem_unmapioregion(uintptr_t phys, u32 size, mali_io_address virt) -+{ -+ iounmap((void *)virt); -+} -+ -+_mali_osk_errcode_t inline _mali_osk_mem_reqregion(uintptr_t phys, u32 size, const char *description) -+{ -+#if MALI_LICENSE_IS_GPL -+ return _MALI_OSK_ERR_OK; /* GPL driver gets the mem region for the resources registered automatically */ -+#else -+ return ((NULL == request_mem_region(phys, size, description)) ? _MALI_OSK_ERR_NOMEM : _MALI_OSK_ERR_OK); -+#endif -+} -+ -+void inline _mali_osk_mem_unreqregion(uintptr_t phys, u32 size) -+{ -+#if !MALI_LICENSE_IS_GPL -+ release_mem_region(phys, size); -+#endif -+} -+ -+void inline _mali_osk_mem_iowrite32_relaxed(volatile mali_io_address addr, u32 offset, u32 val) -+{ -+ __raw_writel(cpu_to_le32(val), ((u8 *)addr) + offset); -+} -+ -+u32 inline _mali_osk_mem_ioread32(volatile mali_io_address addr, u32 offset) -+{ -+ return ioread32(((u8 *)addr) + offset); -+} -+ -+void inline _mali_osk_mem_iowrite32(volatile mali_io_address addr, u32 offset, u32 val) -+{ -+ iowrite32(val, ((u8 *)addr) + offset); -+} -+ -+void _mali_osk_cache_flushall(void) -+{ -+ /** @note Cached memory is not currently supported in this implementation */ -+} -+ -+void _mali_osk_cache_ensure_uncached_range_flushed(void *uncached_mapping, u32 offset, u32 size) -+{ -+ _mali_osk_write_mem_barrier(); -+} -+ -+u32 _mali_osk_mem_write_safe(void __user *dest, const void __user *src, u32 size) -+{ -+#define MALI_MEM_SAFE_COPY_BLOCK_SIZE 4096 -+ u32 retval = 0; -+ void *temp_buf; -+ -+ temp_buf = kmalloc(MALI_MEM_SAFE_COPY_BLOCK_SIZE, GFP_KERNEL); -+ if (NULL != temp_buf) { -+ u32 bytes_left_to_copy = size; -+ u32 i; -+ for (i = 0; i < size; i += MALI_MEM_SAFE_COPY_BLOCK_SIZE) { -+ u32 size_to_copy; -+ u32 size_copied; -+ u32 bytes_left; -+ -+ if (bytes_left_to_copy > MALI_MEM_SAFE_COPY_BLOCK_SIZE) { -+ size_to_copy = MALI_MEM_SAFE_COPY_BLOCK_SIZE; -+ } else { -+ size_to_copy = bytes_left_to_copy; -+ } -+ -+ bytes_left = copy_from_user(temp_buf, ((char *)src) + i, size_to_copy); -+ size_copied = size_to_copy - bytes_left; -+ -+ bytes_left = copy_to_user(((char *)dest) + i, temp_buf, size_copied); -+ size_copied -= bytes_left; -+ -+ bytes_left_to_copy -= size_copied; -+ retval += size_copied; -+ -+ if (size_copied != size_to_copy) { -+ break; /* Early out, we was not able to copy this entire block */ -+ } -+ } -+ -+ kfree(temp_buf); -+ } -+ -+ return retval; -+} -+ -+_mali_osk_errcode_t _mali_ukk_mem_write_safe(_mali_uk_mem_write_safe_s *args) -+{ -+ void __user *src; -+ void __user *dst; -+ struct mali_session_data *session; -+ -+ MALI_DEBUG_ASSERT_POINTER(args); -+ -+ session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ -+ if (NULL == session) { -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+ -+ src = (void __user *)(uintptr_t)args->src; -+ dst = (void __user *)(uintptr_t)args->dest; -+ -+ /* Return number of bytes actually copied */ -+ args->size = _mali_osk_mem_write_safe(dst, src, args->size); -+ return _MALI_OSK_ERR_OK; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c -new file mode 100755 -index 000000000..a729d0499 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_mali.c -@@ -0,0 +1,505 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+ -+/** -+ * @file mali_osk_mali.c -+ * Implementation of the OS abstraction layer which is specific for the Mali kernel device driver -+ */ -+#include "../platform/rk/custom_log.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_osk_mali.h" -+#include "mali_kernel_common.h" /* MALI_xxx macros */ -+#include "mali_osk.h" /* kernel side OS functions */ -+#include "mali_kernel_linux.h" -+ -+static mali_bool mali_secure_mode_enabled = MALI_FALSE; -+static mali_bool mali_secure_mode_supported = MALI_FALSE; -+ -+/* Function that init the mali gpu secure mode */ -+void (*mali_secure_mode_deinit)(void) = NULL; -+/* Function that reset GPU and enable the mali gpu secure mode */ -+int (*mali_gpu_reset_and_secure_mode_enable)(void) = NULL; -+/* Function that reset GPU and disable the mali gpu secure mode */ -+int (*mali_gpu_reset_and_secure_mode_disable)(void) = NULL; -+ -+ -+#ifdef CONFIG_MALI_DT -+ -+#define MALI_OSK_INVALID_RESOURCE_ADDRESS 0xFFFFFFFF -+ -+/** -+ * Define the max number of resource we could have. -+ */ -+#define MALI_OSK_MAX_RESOURCE_NUMBER 27 -+ -+/** -+ * Define the max number of resource with interrupts, and they are -+ * the first 20 elements in array mali_osk_resource_bank. -+ */ -+#define MALI_OSK_RESOURCE_WITH_IRQ_NUMBER 20 -+ -+/** -+ * pp core start and end location in mali_osk_resource_bank array. -+ */ -+#define MALI_OSK_RESOURCE_PP_LOCATION_START 2 -+#define MALI_OSK_RESOURCE_PP_LOCATION_END 17 -+ -+/** -+ * L2 cache start and end location in mali_osk_resource_bank array. -+ */ -+#define MALI_OSK_RESOURCE_L2_LOCATION_START 20 -+#define MALI_OSK_RESOURCE_l2_LOCATION_END 22 -+ -+/** -+ * DMA unit location. -+ */ -+#define MALI_OSK_RESOURCE_DMA_LOCATION 26 -+ -+static _mali_osk_resource_t mali_osk_resource_bank[MALI_OSK_MAX_RESOURCE_NUMBER] = { -+ /*-------------------------------------------------------*/ -+ /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */ -+ /* {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "IRQGP",}, */ -+ {.description = "Mali_GP", .base = MALI_OFFSET_GP, .irq_name = "Mali_GP_IRQ",}, -+ /* {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "IRQGPMMU",}, */ -+ {.description = "Mali_GP_MMU", .base = MALI_OFFSET_GP_MMU, .irq_name = "Mali_GP_MMU_IRQ",}, -+ /* {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "IRQPP0",}, */ -+ {.description = "Mali_PP0", .base = MALI_OFFSET_PP0, .irq_name = "Mali_PP0_IRQ",}, -+ /* {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "IRQPPMMU0",}, */ -+ {.description = "Mali_PP0_MMU", .base = MALI_OFFSET_PP0_MMU, .irq_name = "Mali_PP0_MMU_IRQ",}, -+ /* {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "IRQPP1",}, */ -+ {.description = "Mali_PP1", .base = MALI_OFFSET_PP1, .irq_name = "Mali_PP1_IRQ",}, -+ /* {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "IRQPPMMU1",}, */ -+ {.description = "Mali_PP1_MMU", .base = MALI_OFFSET_PP1_MMU, .irq_name = "Mali_PP1_MMU_IRQ",}, -+ -+ {.description = "Mali_PP2", .base = MALI_OFFSET_PP2, .irq_name = "Mali_PP2_IRQ",}, -+ {.description = "Mali_PP2_MMU", .base = MALI_OFFSET_PP2_MMU, .irq_name = "Mali_PP2_MMU_IRQ",}, -+ {.description = "Mali_PP3", .base = MALI_OFFSET_PP3, .irq_name = "Mali_PP3_IRQ",}, -+ {.description = "Mali_PP3_MMU", .base = MALI_OFFSET_PP3_MMU, .irq_name = "Mali_PP3_MMU_IRQ",}, -+ /*-------------------------------------------------------*/ -+ {.description = "Mali_PP4", .base = MALI_OFFSET_PP4, .irq_name = "IRQPP4",}, -+ {.description = "Mali_PP4_MMU", .base = MALI_OFFSET_PP4_MMU, .irq_name = "IRQPPMMU4",}, -+ {.description = "Mali_PP5", .base = MALI_OFFSET_PP5, .irq_name = "IRQPP5",}, -+ {.description = "Mali_PP5_MMU", .base = MALI_OFFSET_PP5_MMU, .irq_name = "IRQPPMMU5",}, -+ {.description = "Mali_PP6", .base = MALI_OFFSET_PP6, .irq_name = "IRQPP6",}, -+ {.description = "Mali_PP6_MMU", .base = MALI_OFFSET_PP6_MMU, .irq_name = "IRQPPMMU6",}, -+ {.description = "Mali_PP7", .base = MALI_OFFSET_PP7, .irq_name = "IRQPP7",}, -+ {.description = "Mali_PP7_MMU", .base = MALI_OFFSET_PP7_MMU, .irq_name = "IRQPPMMU",}, -+ {.description = "Mali_PP_Broadcast", .base = MALI_OFFSET_PP_BCAST, .irq_name = "IRQPP",}, -+ {.description = "Mali_PMU", .base = MALI_OFFSET_PMU, .irq_name = "IRQPMU",}, -+ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE0,}, -+ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE1,}, -+ {.description = "Mali_L2", .base = MALI_OFFSET_L2_RESOURCE2,}, -+ {.description = "Mali_PP_MMU_Broadcast", .base = MALI_OFFSET_PP_BCAST_MMU,}, -+ {.description = "Mali_Broadcast", .base = MALI_OFFSET_BCAST,}, -+ {.description = "Mali_DLBU", .base = MALI_OFFSET_DLBU,}, -+ {.description = "Mali_DMA", .base = MALI_OFFSET_DMA,}, -+}; -+ -+static int _mali_osk_get_compatible_name(const char **out_string) -+{ -+ struct device_node *node = mali_platform_device->dev.of_node; -+ -+ MALI_DEBUG_ASSERT(NULL != node); -+ -+ return of_property_read_string(node, "compatible", out_string); -+} -+ -+_mali_osk_errcode_t _mali_osk_resource_initialize(void) -+{ -+ mali_bool mali_is_450 = MALI_FALSE, mali_is_470 = MALI_FALSE; -+ int i, pp_core_num = 0, l2_core_num = 0; -+ struct resource *res; -+ const char *compatible_name = NULL; -+ -+ if (0 == _mali_osk_get_compatible_name(&compatible_name)) { -+ if (0 == strncmp(compatible_name, "arm,mali-450", strlen("arm,mali-450"))) { -+ mali_is_450 = MALI_TRUE; -+ MALI_DEBUG_PRINT(2, ("mali-450 device tree detected.")); -+ } else if (0 == strncmp(compatible_name, "arm,mali-470", strlen("arm,mali-470"))) { -+ mali_is_470 = MALI_TRUE; -+ MALI_DEBUG_PRINT(2, ("mali-470 device tree detected.")); -+ } -+ } -+ -+ for (i = 0; i < MALI_OSK_RESOURCE_WITH_IRQ_NUMBER; i++) { -+ res = platform_get_resource_byname(mali_platform_device, IORESOURCE_IRQ, mali_osk_resource_bank[i].irq_name); -+ if (res) { -+ mali_osk_resource_bank[i].irq = res->start; -+ } else { -+ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; -+ } -+ } -+ -+ for (i = MALI_OSK_RESOURCE_PP_LOCATION_START; i <= MALI_OSK_RESOURCE_PP_LOCATION_END; i++) { -+ if (MALI_OSK_INVALID_RESOURCE_ADDRESS != mali_osk_resource_bank[i].base) { -+ pp_core_num++; -+ } -+ } -+ -+ /* We have to divide by 2, because we caculate twice for only one pp(pp_core and pp_mmu_core). */ -+ if (0 != pp_core_num % 2) { -+ MALI_DEBUG_PRINT(2, ("The value of pp core number isn't normal.")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ pp_core_num /= 2; -+ -+ /** -+ * we can caculate the number of l2 cache core according the number of pp core number -+ * and device type(mali400/mali450/mali470). -+ */ -+ l2_core_num = 1; -+ if (mali_is_450) { -+ if (pp_core_num > 4) { -+ l2_core_num = 3; -+ } else if (pp_core_num <= 4) { -+ l2_core_num = 2; -+ } -+ } -+ -+ for (i = MALI_OSK_RESOURCE_l2_LOCATION_END; i > MALI_OSK_RESOURCE_L2_LOCATION_START + l2_core_num - 1; i--) { -+ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; -+ } -+ -+ /* If device is not mali-450 type, we have to remove related resource from resource bank. */ -+ if (!(mali_is_450 || mali_is_470)) { -+ for (i = MALI_OSK_RESOURCE_l2_LOCATION_END + 1; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { -+ mali_osk_resource_bank[i].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; -+ } -+ } -+ -+ if (mali_is_470) -+ mali_osk_resource_bank[MALI_OSK_RESOURCE_DMA_LOCATION].base = MALI_OSK_INVALID_RESOURCE_ADDRESS; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) -+{ -+ int i; -+ -+ if (NULL == mali_platform_device) { -+ return _MALI_OSK_ERR_ITEM_NOT_FOUND; -+ } -+ -+ /* Traverse all of resources in resources bank to find the matching one. */ -+ for (i = 0; i < MALI_OSK_MAX_RESOURCE_NUMBER; i++) { -+ if (mali_osk_resource_bank[i].base == addr) { -+ if (NULL != res) { -+ res->base = addr + _mali_osk_resource_base_address(); -+ res->description = mali_osk_resource_bank[i].description; -+ res->irq = mali_osk_resource_bank[i].irq; -+ } -+ return _MALI_OSK_ERR_OK; -+ } -+ } -+ -+ return _MALI_OSK_ERR_ITEM_NOT_FOUND; -+} -+ -+uintptr_t _mali_osk_resource_base_address(void) -+{ -+ struct resource *reg_res = NULL; -+ uintptr_t ret = 0; -+ -+ reg_res = platform_get_resource(mali_platform_device, IORESOURCE_MEM, 0); -+ -+ if (NULL != reg_res) { -+ ret = reg_res->start; -+ } -+ -+ return ret; -+} -+ -+void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) -+{ -+ struct device_node *node = mali_platform_device->dev.of_node; -+ struct property *prop; -+ const __be32 *p; -+ int length = 0, i = 0; -+ u32 u; -+ -+ MALI_DEBUG_PRINT(2, ("Get pmu config from device tree configuration.\n")); -+ -+ MALI_DEBUG_ASSERT(NULL != node); -+ -+ if (!of_get_property(node, "pmu_domain_config", &length)) { -+ return; -+ } -+ -+ if (array_size != length / sizeof(u32)) { -+ MALI_PRINT_ERROR(("Wrong pmu domain config in device tree.")); -+ return; -+ } -+ -+ of_property_for_each_u32(node, "pmu_domain_config", prop, p, u) { -+ domain_config_array[i] = (u16)u; -+ i++; -+ } -+ -+ return; -+} -+ -+u32 _mali_osk_get_pmu_switch_delay(void) -+{ -+ struct device_node *node = mali_platform_device->dev.of_node; -+ u32 switch_delay; -+ -+ MALI_DEBUG_ASSERT(NULL != node); -+ -+ if (0 == of_property_read_u32(node, "pmu_switch_delay", &switch_delay)) { -+ return switch_delay; -+ } else { -+ MALI_DEBUG_PRINT(2, ("Couldn't find pmu_switch_delay in device tree configuration.\n")); -+ } -+ -+ return 0; -+} -+ -+#else /* CONFIG_MALI_DT */ /* 若未 定义 CONFIG_MALI_DT. */ -+ -+_mali_osk_errcode_t _mali_osk_resource_find(u32 addr, _mali_osk_resource_t *res) -+{ -+ int i; -+ uintptr_t phys_addr; -+ -+ if (NULL == mali_platform_device) { -+ /* Not connected to a device */ -+ return _MALI_OSK_ERR_ITEM_NOT_FOUND; -+ } -+ -+ phys_addr = addr + _mali_osk_resource_base_address(); -+ for (i = 0; i < mali_platform_device->num_resources; i++) { -+ if (IORESOURCE_MEM == resource_type(&(mali_platform_device->resource[i])) && -+ mali_platform_device->resource[i].start == phys_addr) { -+ if (NULL != res) { -+ res->base = phys_addr; -+ res->description = mali_platform_device->resource[i].name; -+ -+ /* Any (optional) IRQ resource belonging to this resource will follow */ -+ if ((i + 1) < mali_platform_device->num_resources && -+ IORESOURCE_IRQ == resource_type(&(mali_platform_device->resource[i + 1]))) { -+ res->irq = mali_platform_device->resource[i + 1].start; -+ } else { -+ res->irq = -1; -+ } -+ } -+ return _MALI_OSK_ERR_OK; -+ } -+ } -+ -+ return _MALI_OSK_ERR_ITEM_NOT_FOUND; -+} -+ -+uintptr_t _mali_osk_resource_base_address(void) -+{ -+ uintptr_t lowest_addr = (uintptr_t)(0 - 1); -+ uintptr_t ret = 0; -+ -+ if (NULL != mali_platform_device) { -+ int i; -+ for (i = 0; i < mali_platform_device->num_resources; i++) { -+ if (mali_platform_device->resource[i].flags & IORESOURCE_MEM && -+ mali_platform_device->resource[i].start < lowest_addr) { -+ lowest_addr = mali_platform_device->resource[i].start; -+ ret = lowest_addr; -+ } -+ } -+ } -+ -+ return ret; -+} -+ -+void _mali_osk_device_data_pmu_config_get(u16 *domain_config_array, int array_size) -+{ -+ _mali_osk_device_data data = { 0, }; -+ -+ MALI_DEBUG_PRINT(2, ("Get pmu config from platform device data.\n")); -+ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { -+ /* Copy the custom customer power domain config */ -+ _mali_osk_memcpy(domain_config_array, data.pmu_domain_config, sizeof(data.pmu_domain_config)); -+ } -+ -+ return; -+} -+ -+u32 _mali_osk_get_pmu_switch_delay(void) -+{ -+ _mali_osk_errcode_t err; -+ _mali_osk_device_data data = { 0, }; -+ -+ err = _mali_osk_device_data_get(&data); -+ -+ if (_MALI_OSK_ERR_OK == err) { -+ return data.pmu_switch_delay; -+ } -+ -+ return 0; -+} -+#endif /* CONFIG_MALI_DT */ -+ -+_mali_osk_errcode_t _mali_osk_device_data_get(_mali_osk_device_data *data) -+{ -+ MALI_DEBUG_ASSERT_POINTER(data); -+ -+ if (NULL != mali_platform_device) { -+ struct mali_gpu_device_data *os_data = NULL; -+ -+ os_data = (struct mali_gpu_device_data *)mali_platform_device->dev.platform_data; -+ if (NULL != os_data) { -+ /* Copy data from OS dependant struct to Mali neutral struct (identical!) */ -+ BUILD_BUG_ON(sizeof(*os_data) != sizeof(*data)); -+ _mali_osk_memcpy(data, os_data, sizeof(*os_data)); -+ -+ return _MALI_OSK_ERR_OK; -+ } -+ } -+ -+ return _MALI_OSK_ERR_ITEM_NOT_FOUND; -+} -+ -+u32 _mali_osk_identify_gpu_resource(void) -+{ -+ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_L2_RESOURCE1, NULL)) -+ /* Mali 450 */ -+ return 0x450; -+ -+ if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_DLBU, NULL)) -+ /* Mali 470 */ -+ return 0x470; -+ -+ /* Mali 400 */ -+ return 0x400; -+} -+ -+mali_bool _mali_osk_shared_interrupts(void) -+{ -+ u32 irqs[128]; -+ u32 i, j, irq, num_irqs_found = 0; -+ -+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); -+ MALI_DEBUG_ASSERT(128 >= mali_platform_device->num_resources); -+ -+ for (i = 0; i < mali_platform_device->num_resources; i++) { -+ if (IORESOURCE_IRQ & mali_platform_device->resource[i].flags) { -+ irq = mali_platform_device->resource[i].start; -+ -+ for (j = 0; j < num_irqs_found; ++j) { -+ if (irq == irqs[j]) { -+ return MALI_TRUE; -+ } -+ } -+ -+ irqs[num_irqs_found++] = irq; -+ } -+ } -+ -+ return MALI_FALSE; -+} -+ -+_mali_osk_errcode_t _mali_osk_gpu_secure_mode_init(void) -+{ -+ _mali_osk_device_data data = { 0, }; -+ -+ if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) { -+ if ((NULL != data.secure_mode_init) && (NULL != data.secure_mode_deinit) -+ && (NULL != data.gpu_reset_and_secure_mode_enable) && (NULL != data.gpu_reset_and_secure_mode_disable)) { -+ int err = data.secure_mode_init(); -+ if (err) { -+ MALI_DEBUG_PRINT(1, ("Failed to init gpu secure mode.\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ mali_secure_mode_deinit = data.secure_mode_deinit; -+ mali_gpu_reset_and_secure_mode_enable = data.gpu_reset_and_secure_mode_enable; -+ mali_gpu_reset_and_secure_mode_disable = data.gpu_reset_and_secure_mode_disable; -+ -+ mali_secure_mode_supported = MALI_TRUE; -+ mali_secure_mode_enabled = MALI_FALSE; -+ return _MALI_OSK_ERR_OK; -+ } -+ } -+ MALI_DEBUG_PRINT(3, ("GPU secure mode not supported.\n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ -+} -+ -+_mali_osk_errcode_t _mali_osk_gpu_secure_mode_deinit(void) -+{ -+ if (NULL != mali_secure_mode_deinit) { -+ mali_secure_mode_deinit(); -+ mali_secure_mode_enabled = MALI_FALSE; -+ mali_secure_mode_supported = MALI_FALSE; -+ return _MALI_OSK_ERR_OK; -+ } -+ MALI_DEBUG_PRINT(3, ("GPU secure mode not supported.\n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ -+} -+ -+ -+_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_enable(void) -+{ -+ /* the mali executor lock must be held before enter this function. */ -+ -+ MALI_DEBUG_ASSERT(MALI_FALSE == mali_secure_mode_enabled); -+ -+ if (NULL != mali_gpu_reset_and_secure_mode_enable) { -+ if (mali_gpu_reset_and_secure_mode_enable()) { -+ MALI_DEBUG_PRINT(1, ("Failed to reset GPU or enable gpu secure mode.\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ mali_secure_mode_enabled = MALI_TRUE; -+ return _MALI_OSK_ERR_OK; -+ } -+ MALI_DEBUG_PRINT(1, ("GPU secure mode not supported.\n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+} -+ -+_mali_osk_errcode_t _mali_osk_gpu_reset_and_secure_mode_disable(void) -+{ -+ /* the mali executor lock must be held before enter this function. */ -+ -+ MALI_DEBUG_ASSERT(MALI_TRUE == mali_secure_mode_enabled); -+ -+ if (NULL != mali_gpu_reset_and_secure_mode_disable) { -+ if (mali_gpu_reset_and_secure_mode_disable()) { -+ MALI_DEBUG_PRINT(1, ("Failed to reset GPU or disable gpu secure mode.\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ mali_secure_mode_enabled = MALI_FALSE; -+ -+ return _MALI_OSK_ERR_OK; -+ -+ } -+ MALI_DEBUG_PRINT(1, ("GPU secure mode not supported.\n")); -+ return _MALI_OSK_ERR_UNSUPPORTED; -+ -+} -+ -+mali_bool _mali_osk_gpu_secure_mode_is_enabled(void) -+{ -+ return mali_secure_mode_enabled; -+} -+ -+mali_bool _mali_osk_gpu_secure_mode_is_supported(void) -+{ -+ return mali_secure_mode_supported; -+} -+ -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c -new file mode 100755 -index 000000000..0b2d00762 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_math.c -@@ -0,0 +1,27 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_math.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include "mali_osk.h" -+#include -+ -+u32 _mali_osk_clz(u32 input) -+{ -+ return 32 - fls(input); -+} -+ -+u32 _mali_osk_fls(u32 input) -+{ -+ return fls(input); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c -new file mode 100755 -index 000000000..174616b56 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_memory.c -@@ -0,0 +1,61 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_memory.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include "mali_osk.h" -+#include -+#include -+ -+void inline *_mali_osk_calloc(u32 n, u32 size) -+{ -+ return kcalloc(n, size, GFP_KERNEL); -+} -+ -+void inline *_mali_osk_malloc(u32 size) -+{ -+ return kmalloc(size, GFP_KERNEL); -+} -+ -+void inline _mali_osk_free(void *ptr) -+{ -+ kfree(ptr); -+} -+ -+void inline *_mali_osk_valloc(u32 size) -+{ -+ return vmalloc(size); -+} -+ -+void inline _mali_osk_vfree(void *ptr) -+{ -+ vfree(ptr); -+} -+ -+void inline *_mali_osk_memcpy(void *dst, const void *src, u32 len) -+{ -+ return memcpy(dst, src, len); -+} -+ -+void inline *_mali_osk_memset(void *s, u32 c, u32 n) -+{ -+ return memset(s, c, n); -+} -+ -+mali_bool _mali_osk_mem_check_allocated(u32 max_allocated) -+{ -+ /* No need to prevent an out-of-memory dialogue appearing on Linux, -+ * so we always return MALI_TRUE. -+ */ -+ return MALI_TRUE; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c -new file mode 100755 -index 000000000..9845187f8 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_misc.c -@@ -0,0 +1,81 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_misc.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include "mali_osk.h" -+ -+#if !defined(CONFIG_MALI_QUIET) -+void _mali_osk_dbgmsg(const char *fmt, ...) -+{ -+ va_list args; -+ va_start(args, fmt); -+ vprintk(fmt, args); -+ va_end(args); -+} -+#endif /* !defined(CONFIG_MALI_QUIET) */ -+ -+u32 _mali_osk_snprintf(char *buf, u32 size, const char *fmt, ...) -+{ -+ int res; -+ va_list args; -+ va_start(args, fmt); -+ -+ res = vscnprintf(buf, (size_t)size, fmt, args); -+ -+ va_end(args); -+ return res; -+} -+ -+void _mali_osk_abort(void) -+{ -+ /* make a simple fault by dereferencing a NULL pointer */ -+ dump_stack(); -+ *(volatile int *)0 = 0; -+} -+ -+void _mali_osk_break(void) -+{ -+ _mali_osk_abort(); -+} -+ -+u32 _mali_osk_get_pid(void) -+{ -+ /* Thread group ID is the process ID on Linux */ -+ return (u32)current->tgid; -+} -+ -+char *_mali_osk_get_comm(void) -+{ -+ return (char *)current->comm; -+} -+ -+ -+u32 _mali_osk_get_tid(void) -+{ -+ /* pid is actually identifying the thread on Linux */ -+ u32 tid = current->pid; -+ -+ /* If the pid is 0 the core was idle. Instead of returning 0 we return a special number -+ * identifying which core we are on. */ -+ if (0 == tid) { -+ tid = -(1 + raw_smp_processor_id()); -+ } -+ -+ return tid; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c -new file mode 100755 -index 000000000..a05f8f066 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_notification.c -@@ -0,0 +1,182 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_notification.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+ -+#include -+#include -+#include -+ -+/** -+ * Declaration of the notification queue object type -+ * Contains a linked list of notification pending delivery to user space. -+ * It also contains a wait queue of exclusive waiters blocked in the ioctl -+ * When a new notification is posted a single thread is resumed. -+ */ -+struct _mali_osk_notification_queue_t_struct { -+ spinlock_t mutex; /**< Mutex protecting the list */ -+ wait_queue_head_t receive_queue; /**< Threads waiting for new entries to the queue */ -+ struct list_head head; /**< List of notifications waiting to be picked up */ -+}; -+ -+typedef struct _mali_osk_notification_wrapper_t_struct { -+ struct list_head list; /**< Internal linked list variable */ -+ _mali_osk_notification_t data; /**< Notification data */ -+} _mali_osk_notification_wrapper_t; -+ -+_mali_osk_notification_queue_t *_mali_osk_notification_queue_init(void) -+{ -+ _mali_osk_notification_queue_t *result; -+ -+ result = (_mali_osk_notification_queue_t *)kmalloc(sizeof(_mali_osk_notification_queue_t), GFP_KERNEL); -+ if (NULL == result) return NULL; -+ -+ spin_lock_init(&result->mutex); -+ init_waitqueue_head(&result->receive_queue); -+ INIT_LIST_HEAD(&result->head); -+ -+ return result; -+} -+ -+_mali_osk_notification_t *_mali_osk_notification_create(u32 type, u32 size) -+{ -+ /* OPT Recycling of notification objects */ -+ _mali_osk_notification_wrapper_t *notification; -+ -+ notification = (_mali_osk_notification_wrapper_t *)kmalloc(sizeof(_mali_osk_notification_wrapper_t) + size, -+ GFP_KERNEL | __GFP_HIGH | __GFP_RETRY_MAYFAIL); -+ if (NULL == notification) { -+ MALI_DEBUG_PRINT(1, ("Failed to create a notification object\n")); -+ return NULL; -+ } -+ -+ /* Init the list */ -+ INIT_LIST_HEAD(¬ification->list); -+ -+ if (0 != size) { -+ notification->data.result_buffer = ((u8 *)notification) + sizeof(_mali_osk_notification_wrapper_t); -+ } else { -+ notification->data.result_buffer = NULL; -+ } -+ -+ /* set up the non-allocating fields */ -+ notification->data.notification_type = type; -+ notification->data.result_buffer_size = size; -+ -+ /* all ok */ -+ return &(notification->data); -+} -+ -+void _mali_osk_notification_delete(_mali_osk_notification_t *object) -+{ -+ _mali_osk_notification_wrapper_t *notification; -+ MALI_DEBUG_ASSERT_POINTER(object); -+ -+ notification = container_of(object, _mali_osk_notification_wrapper_t, data); -+ -+ /* Free the container */ -+ kfree(notification); -+} -+ -+void _mali_osk_notification_queue_term(_mali_osk_notification_queue_t *queue) -+{ -+ _mali_osk_notification_t *result; -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ -+ while (_MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, &result)) { -+ _mali_osk_notification_delete(result); -+ } -+ -+ /* not much to do, just free the memory */ -+ kfree(queue); -+} -+void _mali_osk_notification_queue_send(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t *object) -+{ -+#if defined(MALI_UPPER_HALF_SCHEDULING) -+ unsigned long irq_flags; -+#endif -+ -+ _mali_osk_notification_wrapper_t *notification; -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ MALI_DEBUG_ASSERT_POINTER(object); -+ -+ notification = container_of(object, _mali_osk_notification_wrapper_t, data); -+ -+#if defined(MALI_UPPER_HALF_SCHEDULING) -+ spin_lock_irqsave(&queue->mutex, irq_flags); -+#else -+ spin_lock(&queue->mutex); -+#endif -+ -+ list_add_tail(¬ification->list, &queue->head); -+ -+#if defined(MALI_UPPER_HALF_SCHEDULING) -+ spin_unlock_irqrestore(&queue->mutex, irq_flags); -+#else -+ spin_unlock(&queue->mutex); -+#endif -+ -+ /* and wake up one possible exclusive waiter */ -+ wake_up(&queue->receive_queue); -+} -+ -+_mali_osk_errcode_t _mali_osk_notification_queue_dequeue(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) -+{ -+#if defined(MALI_UPPER_HALF_SCHEDULING) -+ unsigned long irq_flags; -+#endif -+ -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_ITEM_NOT_FOUND; -+ _mali_osk_notification_wrapper_t *wrapper_object; -+ -+#if defined(MALI_UPPER_HALF_SCHEDULING) -+ spin_lock_irqsave(&queue->mutex, irq_flags); -+#else -+ spin_lock(&queue->mutex); -+#endif -+ -+ if (!list_empty(&queue->head)) { -+ wrapper_object = list_entry(queue->head.next, _mali_osk_notification_wrapper_t, list); -+ *result = &(wrapper_object->data); -+ list_del_init(&wrapper_object->list); -+ ret = _MALI_OSK_ERR_OK; -+ } -+ -+#if defined(MALI_UPPER_HALF_SCHEDULING) -+ spin_unlock_irqrestore(&queue->mutex, irq_flags); -+#else -+ spin_unlock(&queue->mutex); -+#endif -+ -+ return ret; -+} -+ -+_mali_osk_errcode_t _mali_osk_notification_queue_receive(_mali_osk_notification_queue_t *queue, _mali_osk_notification_t **result) -+{ -+ /* check input */ -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ MALI_DEBUG_ASSERT_POINTER(result); -+ -+ /* default result */ -+ *result = NULL; -+ -+ if (wait_event_interruptible(queue->receive_queue, -+ _MALI_OSK_ERR_OK == _mali_osk_notification_queue_dequeue(queue, result))) { -+ return _MALI_OSK_ERR_RESTARTSYSCALL; -+ } -+ -+ return _MALI_OSK_ERR_OK; /* all ok */ -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c -new file mode 100755 -index 000000000..e28e2eb21 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_pm.c -@@ -0,0 +1,83 @@ -+/** -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_pm.c -+ * Implementation of the callback functions from common power management -+ */ -+ -+#include -+ -+#include "mali_kernel_linux.h" -+#ifdef CONFIG_PM_RUNTIME -+#include -+#endif /* CONFIG_PM_RUNTIME */ -+#include -+#include -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+ -+/* Can NOT run in atomic context */ -+_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_sync(void) -+{ -+#ifdef CONFIG_PM_RUNTIME -+ int err; -+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); -+ err = pm_runtime_get_sync(&(mali_platform_device->dev)); -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) -+ pm_runtime_mark_last_busy(&(mali_platform_device->dev)); -+#endif -+ if (0 > err) { -+ MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get_sync() returned error code %d\n", err)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+#endif -+ return _MALI_OSK_ERR_OK; -+} -+ -+/* Can run in atomic context */ -+_mali_osk_errcode_t _mali_osk_pm_dev_ref_get_async(void) -+{ -+#ifdef CONFIG_PM_RUNTIME -+ int err; -+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); -+ err = pm_runtime_get(&(mali_platform_device->dev)); -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) -+ pm_runtime_mark_last_busy(&(mali_platform_device->dev)); -+#endif -+ if (0 > err && -EINPROGRESS != err) { -+ MALI_PRINT_ERROR(("Mali OSK PM: pm_runtime_get() returned error code %d\n", err)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+#endif -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+/* Can run in atomic context */ -+void _mali_osk_pm_dev_ref_put(void) -+{ -+#ifdef CONFIG_PM_RUNTIME -+ MALI_DEBUG_ASSERT_POINTER(mali_platform_device); -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) -+ pm_runtime_mark_last_busy(&(mali_platform_device->dev)); -+ pm_runtime_put_autosuspend(&(mali_platform_device->dev)); -+#else -+ pm_runtime_put(&(mali_platform_device->dev)); -+#endif -+#endif -+} -+ -+void _mali_osk_pm_dev_barrier(void) -+{ -+#ifdef CONFIG_PM_RUNTIME -+ pm_runtime_barrier(&(mali_platform_device->dev)); -+#endif -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c -new file mode 100755 -index 000000000..9e977ea4d ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c -@@ -0,0 +1,1282 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_ukk.h" -+#include "mali_uk_types.h" -+#include "mali_osk_profiling.h" -+#include "mali_linux_trace.h" -+#include "mali_gp.h" -+#include "mali_pp.h" -+#include "mali_l2_cache.h" -+#include "mali_user_settings_db.h" -+#include "mali_executor.h" -+#include "mali_memory_manager.h" -+ -+#define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100 -+#define MALI_PROFILING_STREAM_HOLD_TIME 1000000 /*1 ms */ -+ -+#define MALI_PROFILING_STREAM_BUFFER_SIZE (1 << 12) -+#define MALI_PROFILING_STREAM_BUFFER_NUM 100 -+ -+/** -+ * Define the mali profiling stream struct. -+ */ -+typedef struct mali_profiling_stream { -+ u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE]; -+ u32 used_size; -+ struct list_head list; -+} mali_profiling_stream; -+ -+typedef struct mali_profiling_stream_list { -+ spinlock_t spin_lock; -+ struct list_head free_list; -+ struct list_head queue_list; -+} mali_profiling_stream_list; -+ -+static const char mali_name[] = "4xx"; -+static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n"; -+ -+static u32 profiling_sample_rate = 0; -+static u32 first_sw_counter_index = 0; -+ -+static mali_bool l2_cache_counter_if_enabled = MALI_FALSE; -+static u32 num_counters_enabled = 0; -+static u32 mem_counters_enabled = 0; -+ -+static _mali_osk_atomic_t stream_fd_if_used; -+ -+static wait_queue_head_t stream_fd_wait_queue; -+static mali_profiling_counter *global_mali_profiling_counters = NULL; -+static u32 num_global_mali_profiling_counters = 0; -+ -+static mali_profiling_stream_list *global_mali_stream_list = NULL; -+static mali_profiling_stream *mali_counter_stream = NULL; -+static mali_profiling_stream *mali_core_activity_stream = NULL; -+static u64 mali_core_activity_stream_dequeue_time = 0; -+static spinlock_t mali_activity_lock; -+static u32 mali_activity_cores_num = 0; -+static struct hrtimer profiling_sampling_timer; -+ -+const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS; -+const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS; -+ -+static u32 current_profiling_pid = 0; -+ -+static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list) -+{ -+ mali_profiling_stream *profiling_stream, *tmp_profiling_stream; -+ MALI_DEBUG_ASSERT_POINTER(profiling_stream_list); -+ -+ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) { -+ list_del(&profiling_stream->list); -+ kfree(profiling_stream); -+ } -+ -+ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) { -+ list_del(&profiling_stream->list); -+ kfree(profiling_stream); -+ } -+ -+ kfree(profiling_stream_list); -+} -+ -+static void _mali_profiling_global_stream_list_free(void) -+{ -+ mali_profiling_stream *profiling_stream, *tmp_profiling_stream; -+ unsigned long irq_flags; -+ -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); -+ list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) { -+ profiling_stream->used_size = 0; -+ list_move(&profiling_stream->list, &global_mali_stream_list->free_list); -+ } -+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); -+} -+ -+static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream) -+{ -+ unsigned long irq_flags; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK; -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ MALI_DEBUG_ASSERT_POINTER(stream_list); -+ -+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); -+ -+ if (!list_empty(stream_list)) { -+ *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list); -+ list_del_init(&(*new_mali_profiling_stream)->list); -+ } else { -+ ret = _MALI_OSK_ERR_NOMEM; -+ } -+ -+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); -+ -+ return ret; -+} -+ -+static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream) -+{ -+ unsigned long irq_flags; -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ MALI_DEBUG_ASSERT_POINTER(stream_list); -+ -+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); -+ list_add_tail(¤t_mali_profiling_stream->list, stream_list); -+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); -+} -+ -+static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void) -+{ -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ return list_empty(&global_mali_stream_list->queue_list); -+} -+ -+static u32 _mali_profiling_global_stream_queue_list_next_size(void) -+{ -+ unsigned long irq_flags; -+ u32 size = 0; -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ -+ spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags); -+ if (!list_empty(&global_mali_stream_list->queue_list)) { -+ mali_profiling_stream *next_mali_profiling_stream = -+ list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list); -+ size = next_mali_profiling_stream->used_size; -+ } -+ spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags); -+ return size; -+} -+ -+/* The mali profiling stream file operations functions. */ -+static ssize_t _mali_profiling_stream_read( -+ struct file *filp, -+ char __user *buffer, -+ size_t size, -+ loff_t *f_pos); -+ -+static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait); -+ -+static int _mali_profiling_stream_release(struct inode *inode, struct file *filp); -+ -+/* The timeline stream file operations structure. */ -+static const struct file_operations mali_profiling_stream_fops = { -+ .release = _mali_profiling_stream_release, -+ .read = _mali_profiling_stream_read, -+ .poll = _mali_profiling_stream_poll, -+}; -+ -+static ssize_t _mali_profiling_stream_read( -+ struct file *filp, -+ char __user *buffer, -+ size_t size, -+ loff_t *f_pos) -+{ -+ u32 copy_len = 0; -+ mali_profiling_stream *current_mali_profiling_stream; -+ u32 used_size; -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ -+ while (!_mali_profiling_global_stream_queue_list_if_empty()) { -+ used_size = _mali_profiling_global_stream_queue_list_next_size(); -+ if (used_size <= ((u32)size - copy_len)) { -+ current_mali_profiling_stream = NULL; -+ _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list, -+ ¤t_mali_profiling_stream); -+ MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream); -+ if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) { -+ current_mali_profiling_stream->used_size = 0; -+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); -+ return -EFAULT; -+ } -+ copy_len += current_mali_profiling_stream->used_size; -+ current_mali_profiling_stream->used_size = 0; -+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream); -+ } else { -+ break; -+ } -+ } -+ return (ssize_t)copy_len; -+} -+ -+static unsigned int _mali_profiling_stream_poll(struct file *filp, poll_table *wait) -+{ -+ poll_wait(filp, &stream_fd_wait_queue, wait); -+ if (!_mali_profiling_global_stream_queue_list_if_empty()) -+ return POLLIN; -+ return 0; -+} -+ -+static int _mali_profiling_stream_release(struct inode *inode, struct file *filp) -+{ -+ _mali_osk_atomic_init(&stream_fd_if_used, 0); -+ return 0; -+} -+ -+/* The funs for control packet and stream data.*/ -+static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size) -+{ -+ u32 i; -+ -+ for (i = 0; i < sizeof(size); ++i) -+ buf[i] = (size >> 8 * i) & 0xFF; -+} -+ -+static u32 _mali_profiling_get_packet_size(unsigned char *const buf) -+{ -+ u32 i; -+ u32 size = 0; -+ for (i = 0; i < sizeof(size); ++i) -+ size |= (u32)buf[i] << 8 * i; -+ return size; -+} -+ -+static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size) -+{ -+ u64 int_value = 0; -+ u8 shift = 0; -+ u8 byte_value = ~0; -+ -+ while ((byte_value & 0x80) != 0) { -+ if ((*pos) >= packet_size) -+ return -1; -+ byte_value = buf[*pos]; -+ *pos += 1; -+ int_value |= (u32)(byte_value & 0x7f) << shift; -+ shift += 7; -+ } -+ -+ if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) { -+ int_value |= -(1 << shift); -+ } -+ -+ return int_value; -+} -+ -+static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value) -+{ -+ u32 add_bytes = 0; -+ int more = 1; -+ while (more) { -+ /* low order 7 bits of val */ -+ char byte_value = value & 0x7f; -+ value >>= 7; -+ -+ if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) { -+ more = 0; -+ } else { -+ byte_value |= 0x80; -+ } -+ -+ if ((pos + add_bytes) >= buf_size) -+ return 0; -+ buf[pos + add_bytes] = byte_value; -+ add_bytes++; -+ } -+ -+ return add_bytes; -+} -+ -+static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val) -+{ -+ int add_bytes = 0; -+ int more = 1; -+ while (more) { -+ /* low order 7 bits of x */ -+ char byte_value = val & 0x7f; -+ val >>= 7; -+ -+ if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) { -+ more = 0; -+ } else { -+ byte_value |= 0x80; -+ } -+ -+ MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size); -+ buf[pos + add_bytes] = byte_value; -+ add_bytes++; -+ } -+ -+ return add_bytes; -+} -+ -+static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value) -+{ -+ u32 add_size = STREAM_HEADER_SIZE; -+ MALI_DEBUG_ASSERT_POINTER(profiling_stream); -+ MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE); -+ -+ profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE; -+ -+ add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, -+ profiling_stream->used_size + add_size, current_time); -+ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, -+ profiling_stream->used_size + add_size, (s32)0); -+ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, -+ profiling_stream->used_size + add_size, (s32)key); -+ add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE, -+ profiling_stream->used_size + add_size, (s32)counter_value); -+ -+ _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1, -+ add_size - STREAM_HEADER_SIZE); -+ -+ profiling_stream->used_size += add_size; -+} -+ -+/* The callback function for sampling timer.*/ -+static enum hrtimer_restart _mali_profiling_sampling_counters(struct hrtimer *timer) -+{ -+ u32 counter_index; -+ s64 current_time; -+ MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters); -+ MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list); -+ -+ MALI_DEBUG_ASSERT(NULL == mali_counter_stream); -+ if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( -+ &global_mali_stream_list->free_list, &mali_counter_stream)) { -+ -+ MALI_DEBUG_ASSERT_POINTER(mali_counter_stream); -+ MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size); -+ -+ /* Capture l2 cache counter values if enabled */ -+ if (MALI_TRUE == l2_cache_counter_if_enabled) { -+ int i, j = 0; -+ _mali_profiling_l2_counter_values l2_counters_values; -+ _mali_profiling_get_l2_counters(&l2_counters_values); -+ -+ for (i = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) { -+ if (0 == (j % 2)) -+ _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0); -+ else -+ _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1); -+ j++; -+ } -+ } -+ -+ current_time = (s64)_mali_osk_boot_time_get_ns(); -+ -+ /* Add all enabled counter values into stream */ -+ for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) { -+ /* No need to sample these couners here. */ -+ if (global_mali_profiling_counters[counter_index].enabled) { -+ if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER && -+ global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER) -+ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY) -+ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY) -+ || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) { -+ -+ continue; -+ } -+ -+ if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 && -+ global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) { -+ -+ u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value; -+ -+ _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, -+ global_mali_profiling_counters[counter_index].current_counter_value - prev_val); -+ -+ prev_val = global_mali_profiling_counters[counter_index].current_counter_value; -+ -+ global_mali_profiling_counters[counter_index].prev_counter_value = prev_val; -+ } else { -+ -+ if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) { -+ u32 total_alloc_mem = _mali_ukk_report_memory_usage(); -+ global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE; -+ } -+ _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key, -+ global_mali_profiling_counters[counter_index].current_counter_value); -+ if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER) -+ global_mali_profiling_counters[counter_index].current_counter_value = 0; -+ } -+ } -+ } -+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream); -+ mali_counter_stream = NULL; -+ } else { -+ MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); -+ } -+ -+ wake_up_interruptible(&stream_fd_wait_queue); -+ -+ /*Enable the sampling timer again*/ -+ if (0 != num_counters_enabled && 0 != profiling_sample_rate) { -+ hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate)); -+ return HRTIMER_RESTART; -+ } -+ return HRTIMER_NORESTART; -+} -+ -+static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid) -+{ -+ unsigned long irq_flags; -+ -+ spin_lock_irqsave(&mali_activity_lock, irq_flags); -+ if (activity == 0) -+ mali_activity_cores_num--; -+ else -+ mali_activity_cores_num++; -+ spin_unlock_irqrestore(&mali_activity_lock, irq_flags); -+ -+ if (NULL != global_mali_profiling_counters) { -+ int i ; -+ for (i = 0; i < num_global_mali_profiling_counters; i++) { -+ if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { -+ u64 current_time = _mali_osk_boot_time_get_ns(); -+ u32 add_size = STREAM_HEADER_SIZE; -+ -+ if (NULL != mali_core_activity_stream) { -+ if ((mali_core_activity_stream_dequeue_time + MALI_PROFILING_STREAM_HOLD_TIME < current_time) || -+ (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE -+ - mali_core_activity_stream->used_size)) { -+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); -+ mali_core_activity_stream = NULL; -+ wake_up_interruptible(&stream_fd_wait_queue); -+ } -+ } -+ -+ if (NULL == mali_core_activity_stream) { -+ if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue( -+ &global_mali_stream_list->free_list, &mali_core_activity_stream)) { -+ mali_core_activity_stream_dequeue_time = current_time; -+ } else { -+ MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n")); -+ wake_up_interruptible(&stream_fd_wait_queue); -+ break; -+ } -+ -+ } -+ -+ mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY; -+ -+ add_size += _mali_profiling_pack_long(mali_core_activity_stream->data, -+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time); -+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, -+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core); -+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, -+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key); -+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, -+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity); -+ add_size += _mali_profiling_pack_int(mali_core_activity_stream->data, -+ MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid); -+ -+ _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1, -+ add_size - STREAM_HEADER_SIZE); -+ -+ mali_core_activity_stream->used_size += add_size; -+ -+ if (0 == mali_activity_cores_num) { -+ _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream); -+ mali_core_activity_stream = NULL; -+ wake_up_interruptible(&stream_fd_wait_queue); -+ } -+ -+ break; -+ } -+ } -+ } -+} -+ -+static mali_bool _mali_profiling_global_counters_init(void) -+{ -+ int core_id, counter_index, counter_number, counter_id; -+ u32 num_l2_cache_cores; -+ u32 num_pp_cores; -+ u32 num_gp_cores = 1; -+ -+ MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters); -+ num_pp_cores = mali_pp_get_glob_num_pp_cores(); -+ num_l2_cache_cores = mali_l2_cache_core_get_glob_num_l2_cores(); -+ -+ num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores -+ + MALI_PROFILING_SW_COUNTERS_NUM -+ + MALI_PROFILING_SPECIAL_COUNTERS_NUM -+ + MALI_PROFILING_MEM_COUNTERS_NUM; -+ global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter)); -+ -+ if (NULL == global_mali_profiling_counters) -+ return MALI_FALSE; -+ -+ counter_index = 0; -+ /*Vertex processor counters */ -+ for (core_id = 0; core_id < num_gp_cores; core_id ++) { -+ global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id); -+ -+ for (counter_number = 0; counter_number < 2; counter_number++) { -+ counter_index++; -+ global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number); -+ } -+ } -+ -+ /* Fragment processors' counters */ -+ for (core_id = 0; core_id < num_pp_cores; core_id++) { -+ counter_index++; -+ global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id); -+ -+ for (counter_number = 0; counter_number < 2; counter_number++) { -+ counter_index++; -+ global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number); -+ } -+ } -+ -+ /* L2 Cache counters */ -+ for (core_id = 0; core_id < num_l2_cache_cores; core_id++) { -+ for (counter_number = 0; counter_number < 2; counter_number++) { -+ counter_index++; -+ global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number); -+ } -+ } -+ -+ /* Now set up the software counter entries */ -+ for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) { -+ counter_index++; -+ -+ if (0 == first_sw_counter_index) -+ first_sw_counter_index = counter_index; -+ -+ global_mali_profiling_counters[counter_index].counter_id = counter_id; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER); -+ } -+ -+ /* Now set up the special counter entries */ -+ for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) { -+ -+ counter_index++; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", -+ mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]); -+ -+ global_mali_profiling_counters[counter_index].counter_id = counter_id; -+ } -+ -+ /* Now set up the mem counter entries*/ -+ for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) { -+ -+ counter_index++; -+ _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name, -+ sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s", -+ mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]); -+ -+ global_mali_profiling_counters[counter_index].counter_id = counter_id; -+ } -+ -+ MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters); -+ -+ return MALI_TRUE; -+} -+ -+void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable) -+{ -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (NULL != session) { -+ _mali_osk_notification_t *notification; -+ _mali_osk_notification_queue_t *queue; -+ -+ queue = session->ioctl_queue; -+ MALI_DEBUG_ASSERT(NULL != queue); -+ -+ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER, -+ sizeof(_mali_uk_annotate_profiling_mem_counter_s)); -+ -+ if (NULL != notification) { -+ _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer; -+ data->counter_id = counter_id; -+ data->key = key; -+ data->enable = enable; -+ -+ _mali_osk_notification_queue_send(queue, notification); -+ } else { -+ MALI_PRINT_ERROR(("Failed to create notification object!\n")); -+ } -+ } else { -+ MALI_PRINT_ERROR(("Failed to find the right session!\n")); -+ } -+} -+ -+void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable) -+{ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (NULL != session) { -+ _mali_osk_notification_t *notification; -+ _mali_osk_notification_queue_t *queue; -+ -+ queue = session->ioctl_queue; -+ MALI_DEBUG_ASSERT(NULL != queue); -+ -+ notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE, -+ sizeof(_mali_uk_annotate_profiling_enable_s)); -+ -+ if (NULL != notification) { -+ _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer; -+ data->sampling_rate = sampling_rate; -+ data->enable = enable; -+ -+ _mali_osk_notification_queue_send(queue, notification); -+ } else { -+ MALI_PRINT_ERROR(("Failed to create notification object!\n")); -+ } -+ } else { -+ MALI_PRINT_ERROR(("Failed to find the right session!\n")); -+ } -+} -+ -+ -+_mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start) -+{ -+ int i; -+ mali_profiling_stream *new_mali_profiling_stream = NULL; -+ mali_profiling_stream_list *new_mali_profiling_stream_list = NULL; -+ if (MALI_TRUE == auto_start) { -+ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); -+ } -+ -+ /*Init the global_mali_stream_list*/ -+ MALI_DEBUG_ASSERT(NULL == global_mali_stream_list); -+ new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL); -+ -+ if (NULL == new_mali_profiling_stream_list) { -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ spin_lock_init(&new_mali_profiling_stream_list->spin_lock); -+ INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list); -+ INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list); -+ -+ spin_lock_init(&mali_activity_lock); -+ mali_activity_cores_num = 0; -+ -+ for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) { -+ new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL); -+ if (NULL == new_mali_profiling_stream) { -+ _mali_profiling_stream_list_destory(new_mali_profiling_stream_list); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ INIT_LIST_HEAD(&new_mali_profiling_stream->list); -+ new_mali_profiling_stream->used_size = 0; -+ list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list); -+ -+ } -+ -+ _mali_osk_atomic_init(&stream_fd_if_used, 0); -+ init_waitqueue_head(&stream_fd_wait_queue); -+ -+ hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ -+ profiling_sampling_timer.function = _mali_profiling_sampling_counters; -+ -+ global_mali_stream_list = new_mali_profiling_stream_list; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _mali_osk_profiling_term(void) -+{ -+ if (0 != profiling_sample_rate) { -+ hrtimer_cancel(&profiling_sampling_timer); -+ profiling_sample_rate = 0; -+ } -+ _mali_osk_atomic_term(&stream_fd_if_used); -+ -+ if (NULL != global_mali_profiling_counters) { -+ _mali_osk_free(global_mali_profiling_counters); -+ global_mali_profiling_counters = NULL; -+ num_global_mali_profiling_counters = 0; -+ } -+ -+ if (NULL != global_mali_stream_list) { -+ _mali_profiling_stream_list_destory(global_mali_stream_list); -+ global_mali_stream_list = NULL; -+ } -+ -+} -+ -+void _mali_osk_profiling_stop_sampling(u32 pid) -+{ -+ if (pid == current_profiling_pid) { -+ -+ int i; -+ /* Reset all counter states when closing connection.*/ -+ for (i = 0; i < num_global_mali_profiling_counters; ++i) { -+ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); -+ global_mali_profiling_counters[i].enabled = 0; -+ global_mali_profiling_counters[i].prev_counter_value = 0; -+ global_mali_profiling_counters[i].current_counter_value = 0; -+ } -+ l2_cache_counter_if_enabled = MALI_FALSE; -+ num_counters_enabled = 0; -+ mem_counters_enabled = 0; -+ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); -+ _mali_profiling_control(SW_COUNTER_ENABLE, 0); -+ /* Delete sampling timer when closing connection. */ -+ if (0 != profiling_sample_rate) { -+ hrtimer_cancel(&profiling_sampling_timer); -+ profiling_sample_rate = 0; -+ } -+ current_profiling_pid = 0; -+ } -+} -+ -+void _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) -+{ -+ /*Record the freq & volt to global_mali_profiling_counters here. */ -+ if (0 != profiling_sample_rate) { -+ u32 channel; -+ u32 state; -+ channel = (event_id >> 16) & 0xFF; -+ state = ((event_id >> 24) & 0xF) << 24; -+ -+ switch (state) { -+ case MALI_PROFILING_EVENT_TYPE_SINGLE: -+ if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) { -+ u32 reason = (event_id & 0xFFFF); -+ if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) { -+ _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0); -+ _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1); -+ } -+ } -+ break; -+ case MALI_PROFILING_EVENT_TYPE_START: -+ if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { -+ _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1); -+ } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && -+ (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { -+ u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); -+ _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1); -+ } -+ break; -+ case MALI_PROFILING_EVENT_TYPE_STOP: -+ if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) { -+ _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0); -+ } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) && -+ (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) { -+ u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16); -+ _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0); -+ } -+ break; -+ default: -+ break; -+ } -+ } -+ trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4); -+} -+ -+void _mali_osk_profiling_report_sw_counters(u32 *counters) -+{ -+ trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters); -+} -+ -+void _mali_osk_profiling_record_global_counters(int counter_id, u32 value) -+{ -+ if (NULL != global_mali_profiling_counters) { -+ int i ; -+ for (i = 0; i < num_global_mali_profiling_counters; i++) { -+ if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) { -+ global_mali_profiling_counters[i].current_counter_value = value; -+ break; -+ } -+ } -+ } -+} -+ -+_mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args) -+{ -+ /* Always add process and thread identificator in the first two data elements for events from user space */ -+ _mali_osk_profiling_add_event(args->event_id, _mali_osk_get_pid(), _mali_osk_get_tid(), args->data[2], args->data[3], args->data[4]); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args) -+{ -+ u32 *counters = (u32 *)(uintptr_t)args->counters; -+ -+ _mali_osk_profiling_report_sw_counters(counters); -+ -+ if (NULL != global_mali_profiling_counters) { -+ int i; -+ for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) { -+ if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) { -+ global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i); -+ } -+ } -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args) -+{ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) { -+ -+ s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops, -+ session, -+ O_RDONLY | O_CLOEXEC); -+ -+ args->stream_fd = fd; -+ if (0 > fd) { -+ _mali_osk_atomic_dec(&stream_fd_if_used); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ args->stream_fd = fd; -+ } else { -+ _mali_osk_atomic_dec(&stream_fd_if_used); -+ args->stream_fd = -1; -+ return _MALI_OSK_ERR_BUSY; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args) -+{ -+ u32 control_packet_size; -+ u32 output_buffer_size; -+ -+ struct mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx; -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) { -+ MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ control_packet_size = args->control_packet_size; -+ output_buffer_size = args->response_packet_size; -+ -+ if (0 != control_packet_size) { -+ u8 control_type; -+ u8 *control_packet_data; -+ u8 *response_packet_data; -+ u32 version_length = sizeof(utgard_setup_version) - 1; -+ -+ control_packet_data = (u8 *)(uintptr_t)args->control_packet_data; -+ MALI_DEBUG_ASSERT_POINTER(control_packet_data); -+ response_packet_data = (u8 *)(uintptr_t)args->response_packet_data; -+ MALI_DEBUG_ASSERT_POINTER(response_packet_data); -+ -+ /*Decide if need to ignore Utgard setup version.*/ -+ if (control_packet_size >= version_length) { -+ if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) { -+ if (control_packet_size == version_length) { -+ args->response_packet_size = 0; -+ return _MALI_OSK_ERR_OK; -+ } else { -+ control_packet_data += version_length; -+ control_packet_size -= version_length; -+ } -+ } -+ } -+ -+ current_profiling_pid = _mali_osk_get_pid(); -+ -+ control_type = control_packet_data[0]; -+ switch (control_type) { -+ case PACKET_HEADER_COUNTERS_REQUEST: { -+ int i; -+ -+ if (PACKET_HEADER_SIZE > control_packet_size || -+ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { -+ MALI_PRINT_ERROR(("Wrong control packet size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ /* Send supported counters */ -+ if (PACKET_HEADER_SIZE > output_buffer_size) -+ return _MALI_OSK_ERR_FAULT; -+ -+ *response_packet_data = PACKET_HEADER_COUNTERS_ACK; -+ args->response_packet_size = PACKET_HEADER_SIZE; -+ -+ for (i = 0; i < num_global_mali_profiling_counters; ++i) { -+ u32 name_size = strlen(global_mali_profiling_counters[i].counter_name); -+ -+ if ((args->response_packet_size + name_size + 1) > output_buffer_size) { -+ MALI_PRINT_ERROR(("Response packet data is too large..\n")); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ memcpy(response_packet_data + args->response_packet_size, -+ global_mali_profiling_counters[i].counter_name, name_size + 1); -+ -+ args->response_packet_size += (name_size + 1); -+ -+ if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) { -+ args->response_packet_size += _mali_profiling_pack_int(response_packet_data, -+ output_buffer_size, args->response_packet_size, (s32)1); -+ } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) { -+ args->response_packet_size += _mali_profiling_pack_int(response_packet_data, -+ output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores()); -+ } else { -+ args->response_packet_size += _mali_profiling_pack_int(response_packet_data, -+ output_buffer_size, args->response_packet_size, (s32) - 1); -+ } -+ } -+ -+ _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size); -+ break; -+ } -+ -+ case PACKET_HEADER_COUNTERS_ENABLE: { -+ int i; -+ u32 request_pos = PACKET_HEADER_SIZE; -+ mali_bool sw_counter_if_enabled = MALI_FALSE; -+ -+ if (PACKET_HEADER_SIZE > control_packet_size || -+ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { -+ MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ /* Init all counter states before enable requested counters.*/ -+ for (i = 0; i < num_global_mali_profiling_counters; ++i) { -+ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER); -+ global_mali_profiling_counters[i].enabled = 0; -+ global_mali_profiling_counters[i].prev_counter_value = 0; -+ global_mali_profiling_counters[i].current_counter_value = 0; -+ -+ if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && -+ global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { -+ _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0); -+ } -+ } -+ -+ l2_cache_counter_if_enabled = MALI_FALSE; -+ num_counters_enabled = 0; -+ mem_counters_enabled = 0; -+ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0); -+ _mali_profiling_control(SW_COUNTER_ENABLE, 0); -+ _mali_profiling_notification_enable(session, 0, 0); -+ -+ /* Enable requested counters */ -+ while (request_pos < control_packet_size) { -+ u32 begin = request_pos; -+ u32 event; -+ u32 key; -+ -+ /* Check the counter name which should be ended with null */ -+ while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') { -+ ++request_pos; -+ } -+ -+ if (request_pos >= control_packet_size) -+ return _MALI_OSK_ERR_FAULT; -+ -+ ++request_pos; -+ event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); -+ key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); -+ -+ for (i = 0; i < num_global_mali_profiling_counters; ++i) { -+ u32 name_size = strlen((char *)(control_packet_data + begin)); -+ -+ if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) { -+ if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id -+ && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) { -+ sw_counter_if_enabled = MALI_TRUE; -+ _mali_profiling_control(SW_COUNTER_ENABLE, 1); -+ } -+ -+ if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) { -+ _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1); -+ _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff); -+ _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff); -+ } -+ -+ if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER && -+ global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) { -+ _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, -+ key, 1); -+ mem_counters_enabled++; -+ } -+ -+ global_mali_profiling_counters[i].counter_event = event; -+ global_mali_profiling_counters[i].key = key; -+ global_mali_profiling_counters[i].enabled = 1; -+ -+ _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, -+ global_mali_profiling_counters[i].counter_event); -+ num_counters_enabled++; -+ break; -+ } -+ } -+ -+ if (i == num_global_mali_profiling_counters) { -+ MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ } -+ -+ if (PACKET_HEADER_SIZE <= output_buffer_size) { -+ *response_packet_data = PACKET_HEADER_ACK; -+ _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); -+ args->response_packet_size = PACKET_HEADER_SIZE; -+ } else { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ break; -+ } -+ -+ case PACKET_HEADER_START_CAPTURE_VALUE: { -+ u32 live_rate; -+ u32 request_pos = PACKET_HEADER_SIZE; -+ -+ if (PACKET_HEADER_SIZE > control_packet_size || -+ control_packet_size != _mali_profiling_get_packet_size(control_packet_data + 1)) { -+ MALI_PRINT_ERROR(("Wrong control packet size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ /* Read samping rate in nanoseconds and live rate, start capture.*/ -+ profiling_sample_rate = _mali_profiling_read_packet_int(control_packet_data, -+ &request_pos, control_packet_size); -+ -+ live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size); -+ -+ if (PACKET_HEADER_SIZE <= output_buffer_size) { -+ *response_packet_data = PACKET_HEADER_ACK; -+ _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE); -+ args->response_packet_size = PACKET_HEADER_SIZE; -+ } else { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ if (0 != num_counters_enabled && 0 != profiling_sample_rate) { -+ _mali_profiling_global_stream_list_free(); -+ if (mem_counters_enabled > 0) { -+ _mali_profiling_notification_enable(session, profiling_sample_rate, 1); -+ } -+ hrtimer_start(&profiling_sampling_timer, -+ ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000), -+ HRTIMER_MODE_REL_PINNED); -+ } -+ -+ break; -+ } -+ default: -+ MALI_PRINT_ERROR(("Unsupported profiling packet header type %u.\n", control_type)); -+ args->response_packet_size = 0; -+ return _MALI_OSK_ERR_FAULT; -+ } -+ } else { -+ _mali_osk_profiling_stop_sampling(current_profiling_pid); -+ _mali_profiling_notification_enable(session, 0, 0); -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+/** -+ * Called by gator.ko to set HW counters -+ * -+ * @param counter_id The counter ID. -+ * @param event_id Event ID that the counter should count (HW counter value from TRM). -+ * -+ * @return 1 on success, 0 on failure. -+ */ -+int _mali_profiling_set_event(u32 counter_id, s32 event_id) -+{ -+ if (COUNTER_VP_0_C0 == counter_id) { -+ mali_gp_job_set_gp_counter_src0(event_id); -+ } else if (COUNTER_VP_0_C1 == counter_id) { -+ mali_gp_job_set_gp_counter_src1(event_id); -+ } else if (COUNTER_FP_0_C0 <= counter_id && COUNTER_FP_7_C1 >= counter_id) { -+ /* -+ * Two compatibility notes for this function: -+ * -+ * 1) Previously the DDK allowed per core counters. -+ * -+ * This did not make much sense on Mali-450 with the "virtual PP core" concept, -+ * so this option was removed, and only the same pair of HW counters was allowed on all cores, -+ * beginning with r3p2 release. -+ * -+ * Starting with r4p0, it is now possible to set different HW counters for the different sub jobs. -+ * This should be almost the same, since sub job 0 is designed to run on core 0, -+ * sub job 1 on core 1, and so on. -+ * -+ * The scheduling of PP sub jobs is not predictable, and this often led to situations where core 0 ran 2 -+ * sub jobs, while for instance core 1 ran zero. Having the counters set per sub job would thus increase -+ * the predictability of the returned data (as you would be guaranteed data for all the selected HW counters). -+ * -+ * PS: Core scaling needs to be disabled in order to use this reliably (goes for both solutions). -+ * -+ * The framework/#defines with Gator still indicates that the counter is for a particular core, -+ * but this is internally used as a sub job ID instead (no translation needed). -+ * -+ * 2) Global/default vs per sub job counters -+ * -+ * Releases before r3p2 had only per PP core counters. -+ * r3p2 releases had only one set of default/global counters which applied to all PP cores -+ * Starting with r4p0, we have both a set of default/global counters, -+ * and individual counters per sub job (equal to per core). -+ * -+ * To keep compatibility with Gator/DS-5/streamline, the following scheme is used: -+ * -+ * r3p2 release; only counters set for core 0 is handled, -+ * this is applied as the default/global set of counters, and will thus affect all cores. -+ * -+ * r4p0 release; counters set for core 0 is applied as both the global/default set of counters, -+ * and counters for sub job 0. -+ * Counters set for core 1-7 is only applied for the corresponding sub job. -+ * -+ * This should allow the DS-5/Streamline GUI to have a simple mode where it only allows setting the -+ * values for core 0, and thus this will be applied to all PP sub jobs/cores. -+ * Advanced mode will also be supported, where individual pairs of HW counters can be selected. -+ * -+ * The GUI will (until it is updated) still refer to cores instead of sub jobs, but this is probably -+ * something we can live with! -+ * -+ * Mali-450 note: Each job is not divided into a deterministic number of sub jobs, as the HW DLBU -+ * automatically distributes the load between whatever number of cores is available at this particular time. -+ * A normal PP job on Mali-450 is thus considered a single (virtual) job, and it will thus only be possible -+ * to use a single pair of HW counters (even if the job ran on multiple PP cores). -+ * In other words, only the global/default pair of PP HW counters will be used for normal Mali-450 jobs. -+ */ -+ u32 sub_job = (counter_id - COUNTER_FP_0_C0) >> 1; -+ u32 counter_src = (counter_id - COUNTER_FP_0_C0) & 1; -+ if (0 == counter_src) { -+ mali_pp_job_set_pp_counter_sub_job_src0(sub_job, event_id); -+ if (0 == sub_job) { -+ mali_pp_job_set_pp_counter_global_src0(event_id); -+ } -+ } else { -+ mali_pp_job_set_pp_counter_sub_job_src1(sub_job, event_id); -+ if (0 == sub_job) { -+ mali_pp_job_set_pp_counter_global_src1(event_id); -+ } -+ } -+ } else if (COUNTER_L2_0_C0 <= counter_id && COUNTER_L2_2_C1 >= counter_id) { -+ u32 core_id = (counter_id - COUNTER_L2_0_C0) >> 1; -+ struct mali_l2_cache_core *l2_cache_core = mali_l2_cache_core_get_glob_l2_core(core_id); -+ -+ if (NULL != l2_cache_core) { -+ u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1; -+ mali_l2_cache_core_set_counter_src(l2_cache_core, -+ counter_src, event_id); -+ l2_cache_counter_if_enabled = MALI_TRUE; -+ } -+ } else { -+ return 0; /* Failure, unknown event */ -+ } -+ -+ return 1; /* success */ -+} -+ -+/** -+ * Called by gator.ko to retrieve the L2 cache counter values for all L2 cache cores. -+ * The L2 cache counters are unique in that they are polled by gator, rather than being -+ * transmitted via the tracepoint mechanism. -+ * -+ * @param values Pointer to a _mali_profiling_l2_counter_values structure where -+ * the counter sources and values will be output -+ * @return 0 if all went well; otherwise, return the mask with the bits set for the powered off cores -+ */ -+u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values) -+{ -+ u32 l2_cores_num = mali_l2_cache_core_get_glob_num_l2_cores(); -+ u32 i; -+ -+ MALI_DEBUG_ASSERT(l2_cores_num <= 3); -+ -+ for (i = 0; i < l2_cores_num; i++) { -+ struct mali_l2_cache_core *l2_cache = mali_l2_cache_core_get_glob_l2_core(i); -+ -+ if (NULL == l2_cache) { -+ continue; -+ } -+ -+ mali_l2_cache_core_get_counter_values(l2_cache, -+ &values->cores[i].source0, -+ &values->cores[i].value0, -+ &values->cores[i].source1, -+ &values->cores[i].value1); -+ } -+ -+ return 0; -+} -+ -+/** -+ * Called by gator to control the production of profiling information at runtime. -+ */ -+void _mali_profiling_control(u32 action, u32 value) -+{ -+ switch (action) { -+ case FBDUMP_CONTROL_ENABLE: -+ mali_set_user_setting(_MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, (value == 0 ? MALI_FALSE : MALI_TRUE)); -+ break; -+ case FBDUMP_CONTROL_RATE: -+ mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, value); -+ break; -+ case SW_COUNTER_ENABLE: -+ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, value); -+ break; -+ case FBDUMP_CONTROL_RESIZE_FACTOR: -+ mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, value); -+ break; -+ default: -+ break; /* Ignore unimplemented actions */ -+ } -+} -+ -+/** -+ * Called by gator to get mali api version. -+ */ -+u32 _mali_profiling_get_api_version(void) -+{ -+ return MALI_PROFILING_API_VERSION; -+} -+ -+/** -+* Called by gator to get the data about Mali instance in use: -+* product id, version, number of cores -+*/ -+void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values) -+{ -+ values->mali_product_id = (u32)mali_kernel_core_get_product_id(); -+ values->mali_version_major = mali_kernel_core_get_gpu_major_version(); -+ values->mali_version_minor = mali_kernel_core_get_gpu_minor_version(); -+ values->num_of_l2_cores = mali_l2_cache_core_get_glob_num_l2_cores(); -+ values->num_of_fp_cores = mali_executor_get_num_cores_total(); -+ values->num_of_vp_cores = 1; -+} -+ -+ -+EXPORT_SYMBOL(_mali_profiling_set_event); -+EXPORT_SYMBOL(_mali_profiling_get_l2_counters); -+EXPORT_SYMBOL(_mali_profiling_control); -+EXPORT_SYMBOL(_mali_profiling_get_api_version); -+EXPORT_SYMBOL(_mali_profiling_get_mali_version); -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h -new file mode 100755 -index 000000000..af51161f9 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_specific.h -@@ -0,0 +1,74 @@ -+/* -+ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_specific.h -+ * Defines per-OS Kernel level specifics, such as unusual workarounds for -+ * certain OSs. -+ */ -+ -+#ifndef __MALI_OSK_SPECIFIC_H__ -+#define __MALI_OSK_SPECIFIC_H__ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+ -+#include "mali_osk_types.h" -+#include "mali_kernel_linux.h" -+ -+#define MALI_STATIC_INLINE static inline -+#define MALI_NON_STATIC_INLINE inline -+ -+typedef struct dma_pool *mali_dma_pool; -+ -+typedef u32 mali_dma_addr; -+ -+#if MALI_ENABLE_CPU_CYCLES -+/* Reads out the clock cycle performance counter of the current cpu. -+ It is useful for cost-free (2 cycle) measuring of the time spent -+ in a code path. Sample before and after, the diff number of cycles. -+ When the CPU is idle it will not increase this clock counter. -+ It means that the counter is accurate if only spin-locks are used, -+ but mutexes may lead to too low values since the cpu might "idle" -+ waiting for the mutex to become available. -+ The clock source is configured on the CPU during mali module load, -+ but will not give useful output after a CPU has been power cycled. -+ It is therefore important to configure the system to not turn of -+ the cpu cores when using this functionallity.*/ -+static inline unsigned int mali_get_cpu_cyclecount(void) -+{ -+ unsigned int value; -+ /* Reading the CCNT Register - CPU clock counter */ -+ asm volatile("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value)); -+ return value; -+} -+ -+void mali_init_cpu_time_counters(int reset, int enable_divide_by_64); -+#endif -+ -+ -+MALI_STATIC_INLINE u32 _mali_osk_copy_from_user(void *to, void *from, u32 n) -+{ -+ return (u32)copy_from_user(to, from, (unsigned long)n); -+} -+ -+MALI_STATIC_INLINE mali_bool _mali_osk_in_atomic(void) -+{ -+ return in_atomic(); -+} -+ -+#define _mali_osk_put_user(x, ptr) put_user(x, ptr) -+ -+#endif /* __MALI_OSK_SPECIFIC_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c -new file mode 100755 -index 000000000..d295e712a ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_time.c -@@ -0,0 +1,59 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_time.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include "mali_osk.h" -+#include -+#include -+#include -+ -+mali_bool _mali_osk_time_after_eq(unsigned long ticka, unsigned long tickb) -+{ -+ return time_after_eq(ticka, tickb) ? -+ MALI_TRUE : MALI_FALSE; -+} -+ -+unsigned long _mali_osk_time_mstoticks(u32 ms) -+{ -+ return msecs_to_jiffies(ms); -+} -+ -+u32 _mali_osk_time_tickstoms(unsigned long ticks) -+{ -+ return jiffies_to_msecs(ticks); -+} -+ -+unsigned long _mali_osk_time_tickcount(void) -+{ -+ return jiffies; -+} -+ -+void _mali_osk_time_ubusydelay(u32 usecs) -+{ -+ udelay(usecs); -+} -+ -+u64 _mali_osk_time_get_ns(void) -+{ -+ struct timespec64 tsval; -+ ktime_get_real_ts64(&tsval); -+ return (u64)timespec64_to_ns(&tsval); -+} -+ -+u64 _mali_osk_boot_time_get_ns(void) -+{ -+ struct timespec64 tsval; -+ ktime_get_boottime_ts64(&tsval); -+ return (u64)timespec64_to_ns(&tsval); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c -new file mode 100755 -index 000000000..d01c11482 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_timers.c -@@ -0,0 +1,76 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_timers.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include -+#include -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+ -+struct _mali_osk_timer_t_struct { -+ struct timer_list timer; -+}; -+ -+typedef void (*timer_timeout_function_t)(unsigned long); -+ -+_mali_osk_timer_t *_mali_osk_timer_init(_mali_osk_timer_callback_t callback) -+{ -+ _mali_osk_timer_t *t = (_mali_osk_timer_t *)kmalloc(sizeof(_mali_osk_timer_t), GFP_KERNEL); -+ if (NULL != t) -+ timer_setup(&t->timer, -+ (void (*)(struct timer_list *))callback, 0); -+ return t; -+} -+ -+void _mali_osk_timer_add(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+ tim->timer.expires = jiffies + ticks_to_expire; -+ add_timer(&(tim->timer)); -+} -+ -+void _mali_osk_timer_mod(_mali_osk_timer_t *tim, unsigned long ticks_to_expire) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+ mod_timer(&(tim->timer), jiffies + ticks_to_expire); -+} -+ -+void _mali_osk_timer_del(_mali_osk_timer_t *tim) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+ del_timer_sync(&(tim->timer)); -+} -+ -+void _mali_osk_timer_del_async(_mali_osk_timer_t *tim) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+ del_timer(&(tim->timer)); -+} -+ -+mali_bool _mali_osk_timer_pending(_mali_osk_timer_t *tim) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+ return 1 == timer_pending(&(tim->timer)); -+} -+ -+void _mali_osk_timer_setcallback(_mali_osk_timer_t *tim, _mali_osk_timer_callback_t callback, void *data) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+} -+ -+void _mali_osk_timer_term(_mali_osk_timer_t *tim) -+{ -+ MALI_DEBUG_ASSERT_POINTER(tim); -+ kfree(tim); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c -new file mode 100755 -index 000000000..fa12abd3f ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wait_queue.c -@@ -0,0 +1,78 @@ -+/* -+ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_wait_queue.c -+ * Implemenation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+ -+struct _mali_osk_wait_queue_t_struct { -+ wait_queue_head_t wait_queue; -+}; -+ -+_mali_osk_wait_queue_t *_mali_osk_wait_queue_init(void) -+{ -+ _mali_osk_wait_queue_t *ret = NULL; -+ -+ ret = kmalloc(sizeof(_mali_osk_wait_queue_t), GFP_KERNEL); -+ -+ if (NULL == ret) { -+ return ret; -+ } -+ -+ init_waitqueue_head(&ret->wait_queue); -+ MALI_DEBUG_ASSERT(!waitqueue_active(&ret->wait_queue)); -+ -+ return ret; -+} -+ -+void _mali_osk_wait_queue_wait_event(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data) -+{ -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); -+ wait_event(queue->wait_queue, condition(data)); -+} -+ -+void _mali_osk_wait_queue_wait_event_timeout(_mali_osk_wait_queue_t *queue, mali_bool(*condition)(void *), void *data, u32 timeout) -+{ -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ MALI_DEBUG_PRINT(6, ("Adding to wait queue %p\n", queue)); -+ wait_event_timeout(queue->wait_queue, condition(data), _mali_osk_time_mstoticks(timeout)); -+} -+ -+void _mali_osk_wait_queue_wake_up(_mali_osk_wait_queue_t *queue) -+{ -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ -+ /* if queue is empty, don't attempt to wake up its elements */ -+ if (!waitqueue_active(&queue->wait_queue)) return; -+ -+ MALI_DEBUG_PRINT(6, ("Waking up elements in wait queue %p ....\n", queue)); -+ -+ wake_up_all(&queue->wait_queue); -+ -+ MALI_DEBUG_PRINT(6, ("... elements in wait queue %p woken up\n", queue)); -+} -+ -+void _mali_osk_wait_queue_term(_mali_osk_wait_queue_t *queue) -+{ -+ /* Parameter validation */ -+ MALI_DEBUG_ASSERT_POINTER(queue); -+ -+ /* Linux requires no explicit termination of wait queues */ -+ kfree(queue); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c -new file mode 100755 -index 000000000..d5e258a83 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_osk_wq.c -@@ -0,0 +1,240 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_osk_wq.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+#include /* For memory allocation */ -+#include -+#include -+#include -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_kernel_license.h" -+#include "mali_kernel_linux.h" -+ -+typedef struct _mali_osk_wq_work_s { -+ _mali_osk_wq_work_handler_t handler; -+ void *data; -+ mali_bool high_pri; -+ struct work_struct work_handle; -+} mali_osk_wq_work_object_t; -+ -+typedef struct _mali_osk_wq_delayed_work_s { -+ _mali_osk_wq_work_handler_t handler; -+ void *data; -+ struct delayed_work work; -+} mali_osk_wq_delayed_work_object_t; -+ -+#if MALI_LICENSE_IS_GPL -+static struct workqueue_struct *mali_wq_normal = NULL; -+static struct workqueue_struct *mali_wq_high = NULL; -+#endif -+ -+static void _mali_osk_wq_work_func(struct work_struct *work); -+ -+_mali_osk_errcode_t _mali_osk_wq_init(void) -+{ -+#if MALI_LICENSE_IS_GPL -+ MALI_DEBUG_ASSERT(NULL == mali_wq_normal); -+ MALI_DEBUG_ASSERT(NULL == mali_wq_high); -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36) -+ mali_wq_normal = alloc_workqueue("mali", WQ_UNBOUND, 0); -+ mali_wq_high = alloc_workqueue("mali_high_pri", WQ_HIGHPRI | WQ_UNBOUND, 0); -+#else -+ mali_wq_normal = create_workqueue("mali"); -+ mali_wq_high = create_workqueue("mali_high_pri"); -+#endif -+ if (NULL == mali_wq_normal || NULL == mali_wq_high) { -+ MALI_PRINT_ERROR(("Unable to create Mali workqueues\n")); -+ -+ if (mali_wq_normal) destroy_workqueue(mali_wq_normal); -+ if (mali_wq_high) destroy_workqueue(mali_wq_high); -+ -+ mali_wq_normal = NULL; -+ mali_wq_high = NULL; -+ -+ return _MALI_OSK_ERR_FAULT; -+ } -+#endif /* MALI_LICENSE_IS_GPL */ -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _mali_osk_wq_flush(void) -+{ -+#if MALI_LICENSE_IS_GPL -+ flush_workqueue(mali_wq_high); -+ flush_workqueue(mali_wq_normal); -+#else -+ flush_scheduled_work(); -+#endif -+} -+ -+void _mali_osk_wq_term(void) -+{ -+#if MALI_LICENSE_IS_GPL -+ MALI_DEBUG_ASSERT(NULL != mali_wq_normal); -+ MALI_DEBUG_ASSERT(NULL != mali_wq_high); -+ -+ flush_workqueue(mali_wq_normal); -+ destroy_workqueue(mali_wq_normal); -+ -+ flush_workqueue(mali_wq_high); -+ destroy_workqueue(mali_wq_high); -+ -+ mali_wq_normal = NULL; -+ mali_wq_high = NULL; -+#else -+ flush_scheduled_work(); -+#endif -+} -+ -+_mali_osk_wq_work_t *_mali_osk_wq_create_work(_mali_osk_wq_work_handler_t handler, void *data) -+{ -+ mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); -+ -+ if (NULL == work) return NULL; -+ -+ work->handler = handler; -+ work->data = data; -+ work->high_pri = MALI_FALSE; -+ -+ INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); -+ -+ return work; -+} -+ -+_mali_osk_wq_work_t *_mali_osk_wq_create_work_high_pri(_mali_osk_wq_work_handler_t handler, void *data) -+{ -+ mali_osk_wq_work_object_t *work = kmalloc(sizeof(mali_osk_wq_work_object_t), GFP_KERNEL); -+ -+ if (NULL == work) return NULL; -+ -+ work->handler = handler; -+ work->data = data; -+ work->high_pri = MALI_TRUE; -+ -+ INIT_WORK(&work->work_handle, _mali_osk_wq_work_func); -+ -+ return work; -+} -+ -+void _mali_osk_wq_delete_work(_mali_osk_wq_work_t *work) -+{ -+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; -+ _mali_osk_wq_flush(); -+ kfree(work_object); -+} -+ -+void _mali_osk_wq_delete_work_nonflush(_mali_osk_wq_work_t *work) -+{ -+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; -+ kfree(work_object); -+} -+ -+void _mali_osk_wq_schedule_work(_mali_osk_wq_work_t *work) -+{ -+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; -+#if MALI_LICENSE_IS_GPL -+ queue_work(mali_wq_normal, &work_object->work_handle); -+#else -+ schedule_work(&work_object->work_handle); -+#endif -+} -+ -+void _mali_osk_wq_schedule_work_high_pri(_mali_osk_wq_work_t *work) -+{ -+ mali_osk_wq_work_object_t *work_object = (mali_osk_wq_work_object_t *)work; -+#if MALI_LICENSE_IS_GPL -+ queue_work(mali_wq_high, &work_object->work_handle); -+#else -+ schedule_work(&work_object->work_handle); -+#endif -+} -+ -+static void _mali_osk_wq_work_func(struct work_struct *work) -+{ -+ mali_osk_wq_work_object_t *work_object; -+ -+ work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_work_object_t, work_handle); -+ -+#if MALI_LICENSE_IS_GPL -+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36) -+ /* We want highest Dynamic priority of the thread so that the Jobs depending -+ ** on this thread could be scheduled in time. Without this, this thread might -+ ** sometimes need to wait for some threads in user mode to finish its round-robin -+ ** time, causing *bubble* in the Mali pipeline. Thanks to the new implementation -+ ** of high-priority workqueue in new kernel, this only happens in older kernel. -+ */ -+ if (MALI_TRUE == work_object->high_pri) { -+ set_user_nice(current, -19); -+ } -+#endif -+#endif /* MALI_LICENSE_IS_GPL */ -+ -+ work_object->handler(work_object->data); -+} -+ -+static void _mali_osk_wq_delayed_work_func(struct work_struct *work) -+{ -+ mali_osk_wq_delayed_work_object_t *work_object; -+ -+ work_object = _MALI_OSK_CONTAINER_OF(work, mali_osk_wq_delayed_work_object_t, work.work); -+ work_object->handler(work_object->data); -+} -+ -+mali_osk_wq_delayed_work_object_t *_mali_osk_wq_delayed_create_work(_mali_osk_wq_work_handler_t handler, void *data) -+{ -+ mali_osk_wq_delayed_work_object_t *work = kmalloc(sizeof(mali_osk_wq_delayed_work_object_t), GFP_KERNEL); -+ -+ if (NULL == work) return NULL; -+ -+ work->handler = handler; -+ work->data = data; -+ -+ INIT_DELAYED_WORK(&work->work, _mali_osk_wq_delayed_work_func); -+ -+ return work; -+} -+ -+void _mali_osk_wq_delayed_delete_work_nonflush(_mali_osk_wq_delayed_work_t *work) -+{ -+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; -+ kfree(work_object); -+} -+ -+void _mali_osk_wq_delayed_cancel_work_async(_mali_osk_wq_delayed_work_t *work) -+{ -+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; -+ cancel_delayed_work(&work_object->work); -+} -+ -+void _mali_osk_wq_delayed_cancel_work_sync(_mali_osk_wq_delayed_work_t *work) -+{ -+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; -+ cancel_delayed_work_sync(&work_object->work); -+} -+ -+void _mali_osk_wq_delayed_schedule_work(_mali_osk_wq_delayed_work_t *work, u32 delay) -+{ -+ mali_osk_wq_delayed_work_object_t *work_object = (mali_osk_wq_delayed_work_object_t *)work; -+ -+#if MALI_LICENSE_IS_GPL -+ queue_delayed_work(mali_wq_normal, &work_object->work, delay); -+#else -+ schedule_delayed_work(&work_object->work, delay); -+#endif -+ -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c -new file mode 100755 -index 000000000..931d7f07a ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_pmu_power_up_down.c -@@ -0,0 +1,23 @@ -+/** -+ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_pmu_power_up_down.c -+ */ -+ -+#include -+#include "mali_executor.h" -+ -+int mali_perf_set_num_pp_cores(unsigned int num_cores) -+{ -+ return mali_executor_set_perf_level(num_cores, MALI_FALSE); -+} -+ -+EXPORT_SYMBOL(mali_perf_set_num_pp_cores); -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h -new file mode 100755 -index 000000000..4661cac42 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_events.h -@@ -0,0 +1,17 @@ -+/* -+ * Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_PROFILING_EVENTS_H__ -+#define __MALI_PROFILING_EVENTS_H__ -+ -+/* Simple wrapper in order to find the OS specific location of this file */ -+#include -+ -+#endif /* __MALI_PROFILING_EVENTS_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h -new file mode 100755 -index 000000000..6fdaa427c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_gator_api.h -@@ -0,0 +1,17 @@ -+/* -+ * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_PROFILING_GATOR_API_H__ -+#define __MALI_PROFILING_GATOR_API_H__ -+ -+/* Simple wrapper in order to find the OS specific location of this file */ -+#include -+ -+#endif /* __MALI_PROFILING_GATOR_API_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c -new file mode 100755 -index 000000000..c3a526f0a ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.c -@@ -0,0 +1,275 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_osk_mali.h" -+#include "mali_ukk.h" -+#include "mali_timestamp.h" -+#include "mali_osk_profiling.h" -+#include "mali_user_settings_db.h" -+#include "mali_profiling_internal.h" -+ -+typedef struct mali_profiling_entry { -+ u64 timestamp; -+ u32 event_id; -+ u32 data[5]; -+} mali_profiling_entry; -+ -+typedef enum mali_profiling_state { -+ MALI_PROFILING_STATE_UNINITIALIZED, -+ MALI_PROFILING_STATE_IDLE, -+ MALI_PROFILING_STATE_RUNNING, -+ MALI_PROFILING_STATE_RETURN, -+} mali_profiling_state; -+ -+static _mali_osk_mutex_t *lock = NULL; -+static mali_profiling_state prof_state = MALI_PROFILING_STATE_UNINITIALIZED; -+static mali_profiling_entry *profile_entries = NULL; -+static _mali_osk_atomic_t profile_insert_index; -+static u32 profile_mask = 0; -+ -+static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4); -+ -+void probe_mali_timeline_event(void *data, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned -+ int d2, unsigned int d3, unsigned int d4)) -+{ -+ add_event(event_id, d0, d1, d2, d3, d4); -+} -+ -+_mali_osk_errcode_t _mali_internal_profiling_init(mali_bool auto_start) -+{ -+ profile_entries = NULL; -+ profile_mask = 0; -+ _mali_osk_atomic_init(&profile_insert_index, 0); -+ -+ lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_PROFILING); -+ if (NULL == lock) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ prof_state = MALI_PROFILING_STATE_IDLE; -+ -+ if (MALI_TRUE == auto_start) { -+ u32 limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; /* Use maximum buffer size */ -+ -+ mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE); -+ if (_MALI_OSK_ERR_OK != _mali_internal_profiling_start(&limit)) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _mali_internal_profiling_term(void) -+{ -+ u32 count; -+ -+ /* Ensure profiling is stopped */ -+ _mali_internal_profiling_stop(&count); -+ -+ prof_state = MALI_PROFILING_STATE_UNINITIALIZED; -+ -+ if (NULL != profile_entries) { -+ _mali_osk_vfree(profile_entries); -+ profile_entries = NULL; -+ } -+ -+ if (NULL != lock) { -+ _mali_osk_mutex_term(lock); -+ lock = NULL; -+ } -+} -+ -+_mali_osk_errcode_t _mali_internal_profiling_start(u32 *limit) -+{ -+ _mali_osk_errcode_t ret; -+ mali_profiling_entry *new_profile_entries; -+ -+ _mali_osk_mutex_wait(lock); -+ -+ if (MALI_PROFILING_STATE_RUNNING == prof_state) { -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_BUSY; -+ } -+ -+ new_profile_entries = _mali_osk_valloc(*limit * sizeof(mali_profiling_entry)); -+ -+ if (NULL == new_profile_entries) { -+ _mali_osk_mutex_signal(lock); -+ _mali_osk_vfree(new_profile_entries); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ if (MALI_PROFILING_MAX_BUFFER_ENTRIES < *limit) { -+ *limit = MALI_PROFILING_MAX_BUFFER_ENTRIES; -+ } -+ -+ profile_mask = 1; -+ while (profile_mask <= *limit) { -+ profile_mask <<= 1; -+ } -+ profile_mask >>= 1; -+ -+ *limit = profile_mask; -+ -+ profile_mask--; /* turns the power of two into a mask of one less */ -+ -+ if (MALI_PROFILING_STATE_IDLE != prof_state) { -+ _mali_osk_mutex_signal(lock); -+ _mali_osk_vfree(new_profile_entries); -+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ -+ } -+ -+ profile_entries = new_profile_entries; -+ -+ ret = _mali_timestamp_reset(); -+ -+ if (_MALI_OSK_ERR_OK == ret) { -+ prof_state = MALI_PROFILING_STATE_RUNNING; -+ } else { -+ _mali_osk_vfree(profile_entries); -+ profile_entries = NULL; -+ } -+ -+ register_trace_mali_timeline_event(probe_mali_timeline_event, NULL); -+ -+ _mali_osk_mutex_signal(lock); -+ return ret; -+} -+ -+static inline void add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4) -+{ -+ u32 cur_index = (_mali_osk_atomic_inc_return(&profile_insert_index) - 1) & profile_mask; -+ -+ profile_entries[cur_index].timestamp = _mali_timestamp_get(); -+ profile_entries[cur_index].event_id = event_id; -+ profile_entries[cur_index].data[0] = data0; -+ profile_entries[cur_index].data[1] = data1; -+ profile_entries[cur_index].data[2] = data2; -+ profile_entries[cur_index].data[3] = data3; -+ profile_entries[cur_index].data[4] = data4; -+ -+ /* If event is "leave API function", add current memory usage to the event -+ * as data point 4. This is used in timeline profiling to indicate how -+ * much memory was used when leaving a function. */ -+ if (event_id == (MALI_PROFILING_EVENT_TYPE_SINGLE | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_SINGLE_SW_LEAVE_API_FUNC)) { -+ profile_entries[cur_index].data[4] = _mali_ukk_report_memory_usage(); -+ } -+} -+ -+_mali_osk_errcode_t _mali_internal_profiling_stop(u32 *count) -+{ -+ _mali_osk_mutex_wait(lock); -+ -+ if (MALI_PROFILING_STATE_RUNNING != prof_state) { -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ -+ } -+ -+ /* go into return state (user to retreive events), no more events will be added after this */ -+ prof_state = MALI_PROFILING_STATE_RETURN; -+ -+ unregister_trace_mali_timeline_event(probe_mali_timeline_event, NULL); -+ -+ _mali_osk_mutex_signal(lock); -+ -+ tracepoint_synchronize_unregister(); -+ -+ *count = _mali_osk_atomic_read(&profile_insert_index); -+ if (*count > profile_mask) *count = profile_mask; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+u32 _mali_internal_profiling_get_count(void) -+{ -+ u32 retval = 0; -+ -+ _mali_osk_mutex_wait(lock); -+ if (MALI_PROFILING_STATE_RETURN == prof_state) { -+ retval = _mali_osk_atomic_read(&profile_insert_index); -+ if (retval > profile_mask) retval = profile_mask; -+ } -+ _mali_osk_mutex_signal(lock); -+ -+ return retval; -+} -+ -+_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]) -+{ -+ u32 raw_index = _mali_osk_atomic_read(&profile_insert_index); -+ -+ _mali_osk_mutex_wait(lock); -+ -+ if (index < profile_mask) { -+ if ((raw_index & ~profile_mask) != 0) { -+ index += raw_index; -+ index &= profile_mask; -+ } -+ -+ if (prof_state != MALI_PROFILING_STATE_RETURN) { -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ -+ } -+ -+ if (index >= raw_index) { -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ *timestamp = profile_entries[index].timestamp; -+ *event_id = profile_entries[index].event_id; -+ data[0] = profile_entries[index].data[0]; -+ data[1] = profile_entries[index].data[1]; -+ data[2] = profile_entries[index].data[2]; -+ data[3] = profile_entries[index].data[3]; -+ data[4] = profile_entries[index].data[4]; -+ } else { -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _mali_internal_profiling_clear(void) -+{ -+ _mali_osk_mutex_wait(lock); -+ -+ if (MALI_PROFILING_STATE_RETURN != prof_state) { -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_INVALID_ARGS; /* invalid to call this function in this state */ -+ } -+ -+ prof_state = MALI_PROFILING_STATE_IDLE; -+ profile_mask = 0; -+ _mali_osk_atomic_init(&profile_insert_index, 0); -+ -+ if (NULL != profile_entries) { -+ _mali_osk_vfree(profile_entries); -+ profile_entries = NULL; -+ } -+ -+ _mali_osk_mutex_signal(lock); -+ return _MALI_OSK_ERR_OK; -+} -+ -+mali_bool _mali_internal_profiling_is_recording(void) -+{ -+ return prof_state == MALI_PROFILING_STATE_RUNNING ? MALI_TRUE : MALI_FALSE; -+} -+ -+mali_bool _mali_internal_profiling_have_recording(void) -+{ -+ return prof_state == MALI_PROFILING_STATE_RETURN ? MALI_TRUE : MALI_FALSE; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h -new file mode 100755 -index 000000000..f17b45833 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_profiling_internal.h -@@ -0,0 +1,35 @@ -+/* -+ * Copyright (C) 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_PROFILING_INTERNAL_H__ -+#define __MALI_PROFILING_INTERNAL_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include "mali_osk.h" -+ -+int _mali_internal_profiling_init(mali_bool auto_start); -+void _mali_internal_profiling_term(void); -+ -+mali_bool _mali_internal_profiling_is_recording(void); -+mali_bool _mali_internal_profiling_have_recording(void); -+_mali_osk_errcode_t _mali_internal_profiling_clear(void); -+_mali_osk_errcode_t _mali_internal_profiling_get_event(u32 index, u64 *timestamp, u32 *event_id, u32 data[5]); -+u32 _mali_internal_profiling_get_count(void); -+int _mali_internal_profiling_stop(u32 *count); -+int _mali_internal_profiling_start(u32 *limit); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_PROFILING_INTERNAL_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.c b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c -new file mode 100755 -index 000000000..0d98b518f ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.c -@@ -0,0 +1,665 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_sync.h" -+ -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_timeline.h" -+#include "mali_executor.h" -+ -+#include -+#include -+#include -+ -+struct mali_sync_pt { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_pt sync_pt; -+#else -+ struct mali_internal_sync_point sync_pt; -+#endif -+ struct mali_sync_flag *flag; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ -+#else -+ struct mali_internal_sync_timeline *sync_tl; /**< Sync timeline this pt is connected to. */ -+#endif -+}; -+ -+/** -+ * The sync flag is used to connect sync fences to the Mali Timeline system. Sync fences can be -+ * created from a sync flag, and when the flag is signaled, the sync fences will also be signaled. -+ */ -+struct mali_sync_flag { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ -+#else -+ struct mali_internal_sync_timeline *sync_tl; /**< Sync timeline this flag is connected to. */ -+#endif -+ u32 point; /**< Point on timeline. */ -+ int status; /**< 0 if unsignaled, 1 if signaled without error or negative if signaled with error. */ -+ struct kref refcount; /**< Reference count. */ -+}; -+ -+/** -+ * Mali sync timeline is used to connect mali timeline to sync_timeline. -+ * When fence timeout can print more detailed mali timeline system info. -+ */ -+struct mali_sync_timeline_container { -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ struct sync_timeline sync_timeline; -+#else -+ struct mali_internal_sync_timeline sync_timeline; -+#endif -+ struct mali_timeline *timeline; -+}; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) -+#else -+MALI_STATIC_INLINE struct mali_sync_pt *to_mali_sync_pt(struct mali_internal_sync_point *pt) -+#endif -+{ -+ return container_of(pt, struct mali_sync_pt, sync_pt); -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct sync_timeline *sync_tl) -+#else -+MALI_STATIC_INLINE struct mali_sync_timeline_container *to_mali_sync_tl_container(struct mali_internal_sync_timeline *sync_tl) -+#endif -+{ -+ return container_of(sync_tl, struct mali_sync_timeline_container, sync_timeline); -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static int timeline_has_signaled(struct sync_pt *pt) -+#else -+static int timeline_has_signaled(struct mali_internal_sync_point *pt) -+#endif -+{ -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(pt); -+ mpt = to_mali_sync_pt(pt); -+ -+ MALI_DEBUG_ASSERT_POINTER(mpt->flag); -+ -+ return mpt->flag->status; -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static void timeline_free_pt(struct sync_pt *pt) -+#else -+static void timeline_free_pt(struct mali_internal_sync_point *pt) -+#endif -+{ -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(pt); -+ mpt = to_mali_sync_pt(pt); -+ -+ mali_sync_flag_put(mpt->flag); -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static void timeline_release(struct sync_timeline *sync_timeline) -+#else -+static void timeline_release(struct mali_internal_sync_timeline *sync_timeline) -+#endif -+{ -+ struct mali_sync_timeline_container *mali_sync_tl = NULL; -+ struct mali_timeline *mali_tl = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_timeline); -+ -+ mali_sync_tl = to_mali_sync_tl_container(sync_timeline); -+ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); -+ -+ mali_tl = mali_sync_tl->timeline; -+ -+ /* always signaled timeline didn't have mali container */ -+ if (mali_tl) { -+ if (NULL != mali_tl->spinlock) { -+ mali_spinlock_reentrant_term(mali_tl->spinlock); -+ } -+ _mali_osk_free(mali_tl); -+ } -+ -+ module_put(THIS_MODULE); -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static struct sync_pt *timeline_dup(struct sync_pt *pt) -+{ -+ struct mali_sync_pt *mpt, *new_mpt; -+ struct sync_pt *new_pt; -+ MALI_DEBUG_ASSERT_POINTER(pt); -+ -+ mpt = to_mali_sync_pt(pt); -+ -+ new_pt = sync_pt_create(mpt->sync_tl, sizeof(struct mali_sync_pt)); -+ if (NULL == new_pt) return NULL; -+ -+ new_mpt = to_mali_sync_pt(new_pt); -+ -+ mali_sync_flag_get(mpt->flag); -+ new_mpt->flag = mpt->flag; -+ new_mpt->sync_tl = mpt->sync_tl; -+ -+ return new_pt; -+} -+ -+static int timeline_compare(struct sync_pt *pta, struct sync_pt *ptb) -+{ -+ struct mali_sync_pt *mpta; -+ struct mali_sync_pt *mptb; -+ u32 a, b; -+ -+ MALI_DEBUG_ASSERT_POINTER(pta); -+ MALI_DEBUG_ASSERT_POINTER(ptb); -+ mpta = to_mali_sync_pt(pta); -+ mptb = to_mali_sync_pt(ptb); -+ -+ MALI_DEBUG_ASSERT_POINTER(mpta->flag); -+ MALI_DEBUG_ASSERT_POINTER(mptb->flag); -+ -+ a = mpta->flag->point; -+ b = mptb->flag->point; -+ -+ if (a == b) return 0; -+ -+ return ((b - a) < (a - b) ? -1 : 1); -+} -+#endif -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+static void timeline_print_pt(struct seq_file *s, struct sync_pt *sync_pt) -+{ -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(s); -+ MALI_DEBUG_ASSERT_POINTER(sync_pt); -+ -+ mpt = to_mali_sync_pt(sync_pt); -+ -+ /* It is possible this sync point is just under construct, -+ * make sure the flag is valid before accessing it -+ */ -+ if (mpt->flag) { -+ seq_printf(s, "%u", mpt->flag->point); -+ } else { -+ seq_printf(s, "uninitialized"); -+ } -+} -+ -+static void timeline_print_obj(struct seq_file *s, struct sync_timeline *sync_tl) -+{ -+ struct mali_sync_timeline_container *mali_sync_tl = NULL; -+ struct mali_timeline *mali_tl = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_tl); -+ -+ mali_sync_tl = to_mali_sync_tl_container(sync_tl); -+ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); -+ -+ mali_tl = mali_sync_tl->timeline; -+ -+ if (NULL != mali_tl) { -+ seq_printf(s, "oldest (%u) ", mali_tl->point_oldest); -+ seq_printf(s, "next (%u)", mali_tl->point_next); -+ seq_printf(s, "\n"); -+ -+#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) -+ { -+ u32 tid = _mali_osk_get_tid(); -+ struct mali_timeline_system *system = mali_tl->system; -+ -+ mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); -+ if (!mali_tl->destroyed) { -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ mali_timeline_debug_print_timeline(mali_tl, s); -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ } -+ mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); -+ -+ /* dump job queue status and group running status */ -+ mali_executor_status_dump(); -+ } -+#endif -+ } -+} -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static void timeline_pt_value_str(struct sync_pt *pt, char *str, int size) -+{ -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(str); -+ MALI_DEBUG_ASSERT_POINTER(pt); -+ -+ mpt = to_mali_sync_pt(pt); -+ -+ /* It is possible this sync point is just under construct, -+ * make sure the flag is valid before accessing it -+ */ -+ if (mpt->flag) { -+ _mali_osk_snprintf(str, size, "%u", mpt->flag->point); -+ } else { -+ _mali_osk_snprintf(str, size, "uninitialized"); -+ } -+} -+ -+static void timeline_value_str(struct sync_timeline *timeline, char *str, int size) -+{ -+ struct mali_sync_timeline_container *mali_sync_tl = NULL; -+ struct mali_timeline *mali_tl = NULL; -+ -+ MALI_DEBUG_ASSERT_POINTER(timeline); -+ -+ mali_sync_tl = to_mali_sync_tl_container(timeline); -+ MALI_DEBUG_ASSERT_POINTER(mali_sync_tl); -+ -+ mali_tl = mali_sync_tl->timeline; -+ -+ if (NULL != mali_tl) { -+ _mali_osk_snprintf(str, size, "oldest (%u) ", mali_tl->point_oldest); -+ _mali_osk_snprintf(str, size, "next (%u)", mali_tl->point_next); -+ _mali_osk_snprintf(str, size, "\n"); -+ -+#if defined(MALI_TIMELINE_DEBUG_FUNCTIONS) -+ { -+ u32 tid = _mali_osk_get_tid(); -+ struct mali_timeline_system *system = mali_tl->system; -+ -+ mali_spinlock_reentrant_wait(mali_tl->spinlock, tid); -+ if (!mali_tl->destroyed) { -+ mali_spinlock_reentrant_wait(system->spinlock, tid); -+ mali_timeline_debug_direct_print_timeline(mali_tl); -+ mali_spinlock_reentrant_signal(system->spinlock, tid); -+ } -+ mali_spinlock_reentrant_signal(mali_tl->spinlock, tid); -+ -+ /* dump job queue status and group running status */ -+ mali_executor_status_dump(); -+ } -+#endif -+ } -+} -+#else -+static void timeline_print_sync_pt(struct mali_internal_sync_point *sync_pt) -+{ -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_pt); -+ -+ mpt = to_mali_sync_pt(sync_pt); -+ -+ if (mpt->flag) { -+ MALI_DEBUG_PRINT(2, ("mali_internal_sync_pt: %u\n", mpt->flag->point)); -+ } else { -+ MALI_DEBUG_PRINT(2, ("uninitialized\n", mpt->flag->point)); -+ } -+} -+#endif -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+static struct sync_timeline_ops mali_timeline_ops = { -+ .driver_name = "Mali", -+ .dup = timeline_dup, -+ .has_signaled = timeline_has_signaled, -+ .compare = timeline_compare, -+ .free_pt = timeline_free_pt, -+ .release_obj = timeline_release, -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+ .print_pt = timeline_print_pt, -+ .print_obj = timeline_print_obj, -+#else -+ .pt_value_str = timeline_pt_value_str, -+ .timeline_value_str = timeline_value_str, -+#endif -+}; -+ -+struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) -+{ -+ struct sync_timeline *sync_tl; -+ struct mali_sync_timeline_container *mali_sync_tl; -+ -+ sync_tl = sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); -+ if (NULL == sync_tl) return NULL; -+ -+ mali_sync_tl = to_mali_sync_tl_container(sync_tl); -+ mali_sync_tl->timeline = timeline; -+ -+ /* Grab a reference on the module to ensure the callbacks are present -+ * as long some timeline exists. The reference is released when the -+ * timeline is freed. -+ * Since this function is called from a ioctl on an open file we know -+ * we already have a reference, so using __module_get is safe. */ -+ __module_get(THIS_MODULE); -+ -+ return sync_tl; -+} -+ -+s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence) -+{ -+ s32 fd = -1; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) -+ fd = get_unused_fd(); -+#else -+ fd = get_unused_fd_flags(0); -+#endif -+ -+ if (fd < 0) { -+ sync_fence_put(sync_fence); -+ return -1; -+ } -+ sync_fence_install(sync_fence, fd); -+ -+ return fd; -+} -+ -+struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2) -+{ -+ struct sync_fence *sync_fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_fence1); -+ MALI_DEBUG_ASSERT_POINTER(sync_fence1); -+ -+ sync_fence = sync_fence_merge("mali_merge_fence", sync_fence1, sync_fence2); -+ sync_fence_put(sync_fence1); -+ sync_fence_put(sync_fence2); -+ -+ return sync_fence; -+} -+ -+struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl) -+{ -+ struct mali_sync_flag *flag; -+ struct sync_fence *sync_fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_tl); -+ -+ flag = mali_sync_flag_create(sync_tl, 0); -+ if (NULL == flag) return NULL; -+ -+ sync_fence = mali_sync_flag_create_fence(flag); -+ -+ mali_sync_flag_signal(flag, 0); -+ mali_sync_flag_put(flag); -+ -+ return sync_fence; -+} -+ -+struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, mali_timeline_point point) -+{ -+ struct mali_sync_flag *flag; -+ -+ if (NULL == sync_tl) return NULL; -+ -+ flag = _mali_osk_calloc(1, sizeof(*flag)); -+ if (NULL == flag) return NULL; -+ -+ flag->sync_tl = sync_tl; -+ flag->point = point; -+ -+ flag->status = 0; -+ kref_init(&flag->refcount); -+ -+ return flag; -+} -+ -+/** -+ * Create a sync point attached to given sync flag. -+ * -+ * @note Sync points must be triggered in *exactly* the same order as they are created. -+ * -+ * @param flag Sync flag. -+ * @return New sync point if successful, NULL if not. -+ */ -+static struct sync_pt *mali_sync_flag_create_pt(struct mali_sync_flag *flag) -+{ -+ struct sync_pt *pt; -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); -+ -+ pt = sync_pt_create(flag->sync_tl, sizeof(struct mali_sync_pt)); -+ if (NULL == pt) return NULL; -+ -+ mali_sync_flag_get(flag); -+ -+ mpt = to_mali_sync_pt(pt); -+ mpt->flag = flag; -+ mpt->sync_tl = flag->sync_tl; -+ -+ return pt; -+} -+ -+struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) -+{ -+ struct sync_pt *sync_pt; -+ struct sync_fence *sync_fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); -+ -+ sync_pt = mali_sync_flag_create_pt(flag); -+ if (NULL == sync_pt) return NULL; -+ -+ sync_fence = sync_fence_create("mali_flag_fence", sync_pt); -+ if (NULL == sync_fence) { -+ sync_pt_free(sync_pt); -+ return NULL; -+ } -+ -+ return sync_fence; -+} -+#else -+static struct mali_internal_sync_timeline_ops mali_timeline_ops = { -+ .driver_name = "Mali", -+ .has_signaled = timeline_has_signaled, -+ .free_pt = timeline_free_pt, -+ .release_obj = timeline_release, -+ .print_sync_pt = timeline_print_sync_pt, -+}; -+ -+struct mali_internal_sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name) -+{ -+ struct mali_internal_sync_timeline *sync_tl; -+ struct mali_sync_timeline_container *mali_sync_tl; -+ -+ sync_tl = mali_internal_sync_timeline_create(&mali_timeline_ops, sizeof(struct mali_sync_timeline_container), name); -+ if (NULL == sync_tl) return NULL; -+ -+ mali_sync_tl = to_mali_sync_tl_container(sync_tl); -+ mali_sync_tl->timeline = timeline; -+ -+ /* Grab a reference on the module to ensure the callbacks are present -+ * as long some timeline exists. The reference is released when the -+ * timeline is freed. -+ * Since this function is called from a ioctl on an open file we know -+ * we already have a reference, so using __module_get is safe. */ -+ __module_get(THIS_MODULE); -+ -+ return sync_tl; -+} -+ -+s32 mali_sync_fence_fd_alloc(struct mali_internal_sync_fence *sync_fence) -+{ -+ s32 fd = -1; -+ -+ fd = get_unused_fd_flags(0); -+ -+ if (fd < 0) { -+ fput(sync_fence->file); -+ return -1; -+ } -+ fd_install(fd, sync_fence->file); -+ return fd; -+} -+ -+struct mali_internal_sync_fence *mali_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2) -+{ -+ struct mali_internal_sync_fence *sync_fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_fence1); -+ MALI_DEBUG_ASSERT_POINTER(sync_fence1); -+ -+ sync_fence = mali_internal_sync_fence_merge(sync_fence1, sync_fence2); -+ fput(sync_fence1->file); -+ fput(sync_fence2->file); -+ -+ return sync_fence; -+} -+ -+struct mali_internal_sync_fence *mali_sync_timeline_create_signaled_fence(struct mali_internal_sync_timeline *sync_tl) -+{ -+ struct mali_sync_flag *flag; -+ struct mali_internal_sync_fence *sync_fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(sync_tl); -+ -+ flag = mali_sync_flag_create(sync_tl, 0); -+ if (NULL == flag) return NULL; -+ -+ sync_fence = mali_sync_flag_create_fence(flag); -+ -+ mali_sync_flag_signal(flag, 0); -+ mali_sync_flag_put(flag); -+ -+ return sync_fence; -+} -+ -+struct mali_sync_flag *mali_sync_flag_create(struct mali_internal_sync_timeline *sync_tl, mali_timeline_point point) -+{ -+ struct mali_sync_flag *flag; -+ -+ if (NULL == sync_tl) return NULL; -+ -+ flag = _mali_osk_calloc(1, sizeof(*flag)); -+ if (NULL == flag) return NULL; -+ -+ flag->sync_tl = sync_tl; -+ flag->point = point; -+ -+ flag->status = 0; -+ kref_init(&flag->refcount); -+ -+ return flag; -+} -+ -+/** -+ * Create a sync point attached to given sync flag. -+ * -+ * @note Sync points must be triggered in *exactly* the same order as they are created. -+ * -+ * @param flag Sync flag. -+ * @return New sync point if successful, NULL if not. -+ */ -+static struct mali_internal_sync_point *mali_sync_flag_create_pt(struct mali_sync_flag *flag) -+{ -+ struct mali_internal_sync_point *pt; -+ struct mali_sync_pt *mpt; -+ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); -+ -+ pt = mali_internal_sync_point_create(flag->sync_tl, sizeof(struct mali_sync_pt)); -+ -+ if (pt == NULL) { -+ MALI_PRINT_ERROR(("Mali sync: sync_pt creation failed\n")); -+ return NULL; -+ } -+ mali_sync_flag_get(flag); -+ -+ mpt = to_mali_sync_pt(pt); -+ mpt->flag = flag; -+ mpt->sync_tl = flag->sync_tl; -+ -+ return pt; -+} -+ -+struct mali_internal_sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag) -+{ -+ struct mali_internal_sync_point *sync_pt; -+ struct mali_internal_sync_fence *sync_fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ MALI_DEBUG_ASSERT_POINTER(flag->sync_tl); -+ -+ sync_pt = mali_sync_flag_create_pt(flag); -+ if (NULL == sync_pt) { -+ MALI_PRINT_ERROR(("Mali sync: sync_pt creation failed\n")); -+ return NULL; -+ } -+ sync_fence = (struct mali_internal_sync_fence *)sync_file_create(&sync_pt->base); -+ if (NULL == sync_fence) { -+ MALI_PRINT_ERROR(("Mali sync: sync_fence creation failed\n")); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+ dma_fence_put(&sync_pt->base); -+#else -+ fence_put(&sync_pt->base); -+#endif -+ return NULL; -+ } -+ -+ /* 'sync_pt' no longer needs to hold a refcount of '*sync_pt', to put it off. */ -+ dma_fence_put(&sync_pt->base); -+ sync_pt = NULL; -+ -+ return sync_fence; -+} -+#endif -+ -+void mali_sync_flag_get(struct mali_sync_flag *flag) -+{ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ kref_get(&flag->refcount); -+} -+ -+/** -+ * Free sync flag. -+ * -+ * @param ref kref object embedded in sync flag that should be freed. -+ */ -+static void mali_sync_flag_free(struct kref *ref) -+{ -+ struct mali_sync_flag *flag; -+ -+ MALI_DEBUG_ASSERT_POINTER(ref); -+ flag = container_of(ref, struct mali_sync_flag, refcount); -+ -+ _mali_osk_free(flag); -+} -+ -+void mali_sync_flag_put(struct mali_sync_flag *flag) -+{ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ kref_put(&flag->refcount, mali_sync_flag_free); -+} -+ -+void mali_sync_flag_signal(struct mali_sync_flag *flag, int error) -+{ -+ MALI_DEBUG_ASSERT_POINTER(flag); -+ -+ MALI_DEBUG_ASSERT(0 == flag->status); -+ flag->status = (0 > error) ? error : 1; -+ -+ _mali_osk_write_mem_barrier(); -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ sync_timeline_signal(flag->sync_tl); -+#else -+ mali_internal_sync_timeline_signal(flag->sync_tl); -+#endif -+} -+ -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_sync.h b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h -new file mode 100755 -index 000000000..91be8b9cf ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_sync.h -@@ -0,0 +1,169 @@ -+/* -+ * Copyright (C) 2012-2015, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_sync.h -+ * -+ * Mali interface for Linux sync objects. -+ */ -+ -+#ifndef _MALI_SYNC_H_ -+#define _MALI_SYNC_H_ -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ -+#include -+#include -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) -+#include -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+#include -+#else -+#include "mali_internal_sync.h" -+#endif -+ -+ -+#include "mali_osk.h" -+ -+struct mali_sync_flag; -+struct mali_timeline; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+/** -+ * Create a sync timeline. -+ * -+ * @param name Name of the sync timeline. -+ * @return The new sync timeline if successful, NULL if not. -+ */ -+struct sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); -+ -+/** -+ * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of -+ * file descriptor fails. -+ * -+ * @param sync_fence Sync fence. -+ * @return File descriptor representing sync fence if successful, or -1 if not. -+ */ -+s32 mali_sync_fence_fd_alloc(struct sync_fence *sync_fence); -+ -+/** -+ * Merges two sync fences. Both input sync fences will be released. -+ * -+ * @param sync_fence1 First sync fence. -+ * @param sync_fence2 Second sync fence. -+ * @return New sync fence that is the result of the merger if successful, or NULL if not. -+ */ -+struct sync_fence *mali_sync_fence_merge(struct sync_fence *sync_fence1, struct sync_fence *sync_fence2); -+ -+/** -+ * Create a sync fence that is already signaled. -+ * -+ * @param tl Sync timeline. -+ * @return New signaled sync fence if successful, NULL if not. -+ */ -+struct sync_fence *mali_sync_timeline_create_signaled_fence(struct sync_timeline *sync_tl); -+ -+ -+/** -+ * Create a sync flag. -+ * -+ * @param sync_tl Sync timeline. -+ * @param point Point on Mali timeline. -+ * @return New sync flag if successful, NULL if not. -+ */ -+struct mali_sync_flag *mali_sync_flag_create(struct sync_timeline *sync_tl, u32 point); -+ -+/** -+ * Create a sync fence attached to given sync flag. -+ * -+ * @param flag Sync flag. -+ * @return New sync fence if successful, NULL if not. -+ */ -+struct sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); -+#else -+/** -+ * Create a sync timeline. -+ * -+ * @param name Name of the sync timeline. -+ * @return The new sync timeline if successful, NULL if not. -+ */ -+struct mali_internal_sync_timeline *mali_sync_timeline_create(struct mali_timeline *timeline, const char *name); -+ -+/** -+ * Creates a file descriptor representing the sync fence. Will release sync fence if allocation of -+ * file descriptor fails. -+ * -+ * @param sync_fence Sync fence. -+ * @return File descriptor representing sync fence if successful, or -1 if not. -+ */ -+s32 mali_sync_fence_fd_alloc(struct mali_internal_sync_fence *sync_fence); -+ -+/** -+ * Merges two sync fences. Both input sync fences will be released. -+ * -+ * @param sync_fence1 First sync fence. -+ * @param sync_fence2 Second sync fence. -+ * @return New sync fence that is the result of the merger if successful, or NULL if not. -+ */ -+struct mali_internal_sync_fence *mali_sync_fence_merge(struct mali_internal_sync_fence *sync_fence1, struct mali_internal_sync_fence *sync_fence2); -+ -+/** -+ * Create a sync fence that is already signaled. -+ * -+ * @param tl Sync timeline. -+ * @return New signaled sync fence if successful, NULL if not. -+ */ -+struct mali_internal_sync_fence *mali_sync_timeline_create_signaled_fence(struct mali_internal_sync_timeline *sync_tl); -+ -+ -+/** -+ * Create a sync flag. -+ * -+ * @param sync_tl Sync timeline. -+ * @param point Point on Mali timeline. -+ * @return New sync flag if successful, NULL if not. -+ */ -+struct mali_sync_flag *mali_sync_flag_create(struct mali_internal_sync_timeline *sync_tl, u32 point); -+ -+/** -+ * Create a sync fence attached to given sync flag. -+ * -+ * @param flag Sync flag. -+ * @return New sync fence if successful, NULL if not. -+ */ -+struct mali_internal_sync_fence *mali_sync_flag_create_fence(struct mali_sync_flag *flag); -+ -+#endif -+/** -+ * Grab sync flag reference. -+ * -+ * @param flag Sync flag. -+ */ -+void mali_sync_flag_get(struct mali_sync_flag *flag); -+ -+/** -+ * Release sync flag reference. If this was the last reference, the sync flag will be freed. -+ * -+ * @param flag Sync flag. -+ */ -+void mali_sync_flag_put(struct mali_sync_flag *flag); -+ -+/** -+ * Signal sync flag. All sync fences created from this flag will be signaled. -+ * -+ * @param flag Sync flag to signal. -+ * @param error Negative error code, or 0 if no error. -+ */ -+void mali_sync_flag_signal(struct mali_sync_flag *flag, int error); -+ -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+#endif /* _MALI_SYNC_H_ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h -new file mode 100755 -index 000000000..68b27b8be ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_uk_types.h -@@ -0,0 +1,17 @@ -+/* -+ * Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_UK_TYPES_H__ -+#define __MALI_UK_TYPES_H__ -+ -+/* Simple wrapper in order to find the OS specific location of this file */ -+#include -+ -+#endif /* __MALI_UK_TYPES_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c -new file mode 100755 -index 000000000..0bd1cddb1 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_core.c -@@ -0,0 +1,171 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* memort allocation functions */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs) -+{ -+ _mali_uk_get_api_version_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_get_api_version(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; -+ if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; -+ -+ return 0; -+} -+ -+int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs) -+{ -+ _mali_uk_get_api_version_v2_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != get_user(kargs.version, &uargs->version)) return -EFAULT; -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_get_api_version_v2(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; -+ if (0 != put_user(kargs.compatible, &uargs->compatible)) return -EFAULT; -+ -+ return 0; -+} -+ -+/* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ -+#if 0 -+#define mali400_in_rk30_version 0x01 -+int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _mali_uk_get_mali_version_in_rk30_s __user *uargs) -+{ -+ _mali_uk_get_mali_version_in_rk30_s kargs; -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ kargs.ctx = (uintptr_t)session_data; -+ kargs.version = mali400_in_rk30_version; -+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; -+ return 0; -+} -+#else -+#include "../platform/rk/rk_ext.h" -+int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs) -+{ -+ _mali_rk_ko_version_s kargs; -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ kargs.ctx = (uintptr_t)session_data; -+ kargs.version = RK_KO_VER; -+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; -+ return 0; -+} -+#endif -+ -+int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs) -+{ -+ _mali_uk_wait_for_notification_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_wait_for_notification(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ if (_MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS != kargs.type) { -+ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ -+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_wait_for_notification_s))) return -EFAULT; -+ } else { -+ if (0 != put_user(kargs.type, &uargs->type)) return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs) -+{ -+ _mali_uk_post_notification_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ -+ if (0 != get_user(kargs.type, &uargs->type)) { -+ return -EFAULT; -+ } -+ -+ err = _mali_ukk_post_notification(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -+int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs) -+{ -+ _mali_uk_get_user_settings_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_get_user_settings(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ kargs.ctx = 0; /* prevent kernel address to be returned to user space */ -+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_user_settings_s))) return -EFAULT; -+ -+ return 0; -+} -+ -+int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs) -+{ -+ _mali_uk_request_high_priority_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_request_high_priority(&kargs); -+ -+ kargs.ctx = 0; -+ -+ return map_errcode(err); -+} -+ -+int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs) -+{ -+ _mali_uk_pending_submit_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_pending_submit(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c -new file mode 100755 -index 000000000..68fcd9719 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_gp.c -@@ -0,0 +1,91 @@ -+/* -+ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs) -+{ -+ _mali_osk_errcode_t err; -+ -+ /* If the job was started successfully, 0 is returned. If there was an error, but the job -+ * was started, we return -ENOENT. For anything else returned, the job was not started. */ -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ err = _mali_ukk_gp_start_job(session_data, uargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ return 0; -+} -+ -+int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs) -+{ -+ _mali_uk_get_gp_core_version_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_get_gp_core_version(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ /* no known transactions to roll-back */ -+ -+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; -+ -+ return 0; -+} -+ -+int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs) -+{ -+ _mali_uk_gp_suspend_response_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_gp_suspend_response_s))) return -EFAULT; -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_gp_suspend_response(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ if (0 != put_user(kargs.cookie, &uargs->cookie)) return -EFAULT; -+ -+ /* no known transactions to roll-back */ -+ return 0; -+} -+ -+int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs) -+{ -+ _mali_uk_get_gp_number_of_cores_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_get_gp_number_of_cores(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ /* no known transactions to roll-back */ -+ -+ if (0 != put_user(kargs.number_of_cores, &uargs->number_of_cores)) return -EFAULT; -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c -new file mode 100755 -index 000000000..baea4c688 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_mem.c -@@ -0,0 +1,333 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs) -+{ -+ _mali_uk_alloc_mem_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_alloc_mem_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_allocate(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs) -+{ -+ _mali_uk_free_mem_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_free_mem_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_free(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ if (0 != put_user(kargs.free_pages_nr, &uargs->free_pages_nr)) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs) -+{ -+ _mali_uk_bind_mem_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_bind_mem_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_bind(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -+int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs) -+{ -+ _mali_uk_unbind_mem_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_unbind_mem_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_unbind(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -+ -+int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs) -+{ -+ _mali_uk_cow_mem_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_mem_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_cow(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ if (0 != put_user(kargs.backend_handle, &uargs->backend_handle)) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs) -+{ -+ _mali_uk_cow_modify_range_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_cow_modify_range_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_cow_modify_range(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ if (0 != put_user(kargs.change_pages_nr, &uargs->change_pages_nr)) { -+ return -EFAULT; -+ } -+ return 0; -+} -+ -+ -+int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs) -+{ -+ _mali_uk_mem_resize_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_resize_s))) { -+ return -EFAULT; -+ } -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_mem_resize(&kargs); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -+int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs) -+{ -+ _mali_uk_mem_write_safe_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_mem_write_safe_s))) { -+ return -EFAULT; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ -+ /* Check if we can access the buffers */ -+ if (!access_ok((const void *)(uintptr_t)kargs.dest, kargs.size) || -+ !access_ok((const void *)(uintptr_t)kargs.src, kargs.size)) { -+ return -EINVAL; -+ } -+ -+ /* Check if size wraps */ -+ if ((kargs.size + kargs.dest) <= kargs.dest -+ || (kargs.size + kargs.src) <= kargs.src) { -+ return -EINVAL; -+ } -+ -+ err = _mali_ukk_mem_write_safe(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ if (0 != put_user(kargs.size, &uargs->size)) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+ -+ -+int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs) -+{ -+ _mali_uk_query_mmu_page_table_dump_size_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_query_mmu_page_table_dump_size(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ if (0 != put_user(kargs.size, &uargs->size)) return -EFAULT; -+ -+ return 0; -+} -+ -+int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs) -+{ -+ _mali_uk_dump_mmu_page_table_s kargs; -+ _mali_osk_errcode_t err; -+ void __user *user_buffer; -+ void *buffer = NULL; -+ int rc = -EFAULT; -+ -+ /* validate input */ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ /* the session_data pointer was validated by caller */ -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_dump_mmu_page_table_s))) -+ goto err_exit; -+ -+ user_buffer = (void __user *)(uintptr_t)kargs.buffer; -+ if (!access_ok(user_buffer, kargs.size)) -+ goto err_exit; -+ -+ /* allocate temporary buffer (kernel side) to store mmu page table info */ -+ if (kargs.size <= 0) -+ return -EINVAL; -+ /* Allow at most 8MiB buffers, this is more than enough to dump a fully -+ * populated page table. */ -+ if (kargs.size > SZ_8M) -+ return -EINVAL; -+ -+ buffer = (void *)(uintptr_t)_mali_osk_valloc(kargs.size); -+ if (NULL == buffer) { -+ rc = -ENOMEM; -+ goto err_exit; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ kargs.buffer = (uintptr_t)buffer; -+ err = _mali_ukk_dump_mmu_page_table(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ rc = map_errcode(err); -+ goto err_exit; -+ } -+ -+ /* copy mmu page table info back to user space and update pointers */ -+ if (0 != copy_to_user(user_buffer, buffer, kargs.size)) -+ goto err_exit; -+ -+ kargs.register_writes = kargs.register_writes - -+ (uintptr_t)buffer + (uintptr_t)user_buffer; -+ kargs.page_table_dump = kargs.page_table_dump - -+ (uintptr_t)buffer + (uintptr_t)user_buffer; -+ -+ if (0 != copy_to_user(uargs, &kargs, sizeof(kargs))) -+ goto err_exit; -+ -+ rc = 0; -+ -+err_exit: -+ if (buffer) _mali_osk_vfree(buffer); -+ return rc; -+} -+ -+int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs) -+{ -+ _mali_osk_errcode_t err; -+ _mali_uk_profiling_memory_usage_get_s kargs; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { -+ return -EFAULT; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_mem_usage_get(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ -+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_memory_usage_get_s))) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c -new file mode 100755 -index 000000000..a9b0958c0 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_pp.c -@@ -0,0 +1,105 @@ -+/* -+ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs) -+{ -+ _mali_osk_errcode_t err; -+ -+ /* If the job was started successfully, 0 is returned. If there was an error, but the job -+ * was started, we return -ENOENT. For anything else returned, the job was not started. */ -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ err = _mali_ukk_pp_start_job(session_data, uargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ return 0; -+} -+ -+int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs) -+{ -+ _mali_osk_errcode_t err; -+ -+ /* If the jobs were started successfully, 0 is returned. If there was an error, but the -+ * jobs were started, we return -ENOENT. For anything else returned, the jobs were not -+ * started. */ -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ err = _mali_ukk_pp_and_gp_start_job(session_data, uargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ return 0; -+} -+ -+int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs) -+{ -+ _mali_uk_get_pp_number_of_cores_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ -+ err = _mali_ukk_get_pp_number_of_cores(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ kargs.ctx = (uintptr_t)NULL; /* prevent kernel address to be returned to user space */ -+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_get_pp_number_of_cores_s))) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs) -+{ -+ _mali_uk_get_pp_core_version_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_get_pp_core_version(&kargs); -+ if (_MALI_OSK_ERR_OK != err) return map_errcode(err); -+ -+ if (0 != put_user(kargs.version, &uargs->version)) return -EFAULT; -+ -+ return 0; -+} -+ -+int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs) -+{ -+ _mali_uk_pp_disable_wb_s kargs; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session_data, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_pp_disable_wb_s))) return -EFAULT; -+ -+ kargs.ctx = (uintptr_t)session_data; -+ _mali_ukk_pp_job_disable_wb(&kargs); -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c -new file mode 100755 -index 000000000..8b49ebc50 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_profiling.c -@@ -0,0 +1,183 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+#include -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs) -+{ -+ _mali_uk_profiling_add_event_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_add_event_s))) { -+ return -EFAULT; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_profiling_add_event(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -+int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs) -+{ -+ _mali_uk_sw_counters_report_s kargs; -+ _mali_osk_errcode_t err; -+ u32 *counter_buffer; -+ u32 __user *counters; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_sw_counters_report_s))) { -+ return -EFAULT; -+ } -+ -+ /* make sure that kargs.num_counters is [at least somewhat] sane */ -+ if (kargs.num_counters > 10000) { -+ MALI_DEBUG_PRINT(1, ("User space attempted to allocate too many counters.\n")); -+ return -EINVAL; -+ } -+ -+ counter_buffer = (u32 *)kmalloc(sizeof(u32) * kargs.num_counters, GFP_KERNEL); -+ if (NULL == counter_buffer) { -+ return -ENOMEM; -+ } -+ -+ counters = (u32 *)(uintptr_t)kargs.counters; -+ -+ if (0 != copy_from_user(counter_buffer, counters, sizeof(u32) * kargs.num_counters)) { -+ kfree(counter_buffer); -+ return -EFAULT; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ kargs.counters = (uintptr_t)counter_buffer; -+ -+ err = _mali_ukk_sw_counters_report(&kargs); -+ -+ kfree(counter_buffer); -+ -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -+int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs) -+{ -+ _mali_uk_profiling_stream_fd_get_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { -+ return -EFAULT; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_profiling_stream_fd_get(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ if (0 != copy_to_user(uargs, &kargs, sizeof(_mali_uk_profiling_stream_fd_get_s))) { -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs) -+{ -+ _mali_uk_profiling_control_set_s kargs; -+ _mali_osk_errcode_t err; -+ u8 *kernel_control_data = NULL; -+ u8 *kernel_response_data = NULL; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != get_user(kargs.control_packet_size, &uargs->control_packet_size)) return -EFAULT; -+ if (0 != get_user(kargs.response_packet_size, &uargs->response_packet_size)) return -EFAULT; -+ -+ kargs.ctx = (uintptr_t)session_data; -+ -+ -+ /* Sanity check about the size */ -+ if (kargs.control_packet_size > PAGE_SIZE || kargs.response_packet_size > PAGE_SIZE) -+ return -EINVAL; -+ -+ if (0 != kargs.control_packet_size) { -+ -+ if (0 == kargs.response_packet_size) -+ return -EINVAL; -+ -+ kernel_control_data = _mali_osk_calloc(1, kargs.control_packet_size); -+ if (NULL == kernel_control_data) { -+ return -ENOMEM; -+ } -+ -+ kernel_response_data = _mali_osk_calloc(1, kargs.response_packet_size); -+ if (NULL == kernel_response_data) { -+ _mali_osk_free(kernel_control_data); -+ return -ENOMEM; -+ } -+ -+ kargs.control_packet_data = (uintptr_t)kernel_control_data; -+ kargs.response_packet_data = (uintptr_t)kernel_response_data; -+ -+ if (0 != copy_from_user((void *)(uintptr_t)kernel_control_data, (void *)(uintptr_t)uargs->control_packet_data, kargs.control_packet_size)) { -+ _mali_osk_free(kernel_control_data); -+ _mali_osk_free(kernel_response_data); -+ return -EFAULT; -+ } -+ -+ err = _mali_ukk_profiling_control_set(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ _mali_osk_free(kernel_control_data); -+ _mali_osk_free(kernel_response_data); -+ return map_errcode(err); -+ } -+ -+ if (0 != kargs.response_packet_size && 0 != copy_to_user(((void *)(uintptr_t)uargs->response_packet_data), ((void *)(uintptr_t)kargs.response_packet_data), kargs.response_packet_size)) { -+ _mali_osk_free(kernel_control_data); -+ _mali_osk_free(kernel_response_data); -+ return -EFAULT; -+ } -+ -+ if (0 != put_user(kargs.response_packet_size, &uargs->response_packet_size)) { -+ _mali_osk_free(kernel_control_data); -+ _mali_osk_free(kernel_response_data); -+ return -EFAULT; -+ } -+ -+ _mali_osk_free(kernel_control_data); -+ _mali_osk_free(kernel_response_data); -+ } else { -+ -+ err = _mali_ukk_profiling_control_set(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ } -+ return 0; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c -new file mode 100755 -index 000000000..1dd4a7c6f ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_soft_job.c -@@ -0,0 +1,90 @@ -+/* -+ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+#include "mali_soft_job.h" -+#include "mali_timeline.h" -+ -+int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs) -+{ -+ _mali_uk_soft_job_start_s kargs; -+ u32 type, point; -+ u64 user_job; -+ struct mali_timeline_fence fence; -+ struct mali_soft_job *job = NULL; -+ u32 __user *job_id_ptr = NULL; -+ -+ /* If the job was started successfully, 0 is returned. If there was an error, but the job -+ * was started, we return -ENOENT. For anything else returned, the job was not started. */ -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ MALI_CHECK_NON_NULL(session, -EINVAL); -+ -+ MALI_DEBUG_ASSERT_POINTER(session->soft_job_system); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(kargs))) { -+ return -EFAULT; -+ } -+ -+ type = kargs.type; -+ user_job = kargs.user_job; -+ job_id_ptr = (u32 __user *)(uintptr_t)kargs.job_id_ptr; -+ -+ mali_timeline_fence_copy_uk_fence(&fence, &kargs.fence); -+ -+ if ((MALI_SOFT_JOB_TYPE_USER_SIGNALED != type) && (MALI_SOFT_JOB_TYPE_SELF_SIGNALED != type)) { -+ MALI_DEBUG_PRINT_ERROR(("Invalid soft job type specified\n")); -+ return -EINVAL; -+ } -+ -+ /* Create soft job. */ -+ job = mali_soft_job_create(session->soft_job_system, (enum mali_soft_job_type)type, user_job); -+ if (unlikely(NULL == job)) { -+ return map_errcode(_MALI_OSK_ERR_NOMEM); -+ } -+ -+ /* Write job id back to user space. */ -+ if (0 != put_user(job->id, job_id_ptr)) { -+ MALI_PRINT_ERROR(("Mali Soft Job: failed to put job id")); -+ mali_soft_job_destroy(job); -+ return map_errcode(_MALI_OSK_ERR_NOMEM); -+ } -+ -+ /* Start soft job. */ -+ point = mali_soft_job_start(job, &fence); -+ -+ if (0 != put_user(point, &uargs->point)) { -+ /* Let user space know that something failed after the job was started. */ -+ return -ENOENT; -+ } -+ -+ return 0; -+} -+ -+int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs) -+{ -+ u32 job_id; -+ _mali_osk_errcode_t err; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (0 != get_user(job_id, &uargs->job_id)) return -EFAULT; -+ -+ err = mali_soft_job_system_signal_job(session->soft_job_system, job_id); -+ -+ return map_errcode(err); -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c -new file mode 100755 -index 000000000..ff0c90939 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_timeline.c -@@ -0,0 +1,88 @@ -+/* -+ * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+#include "mali_timeline.h" -+#include "mali_timeline_fence_wait.h" -+#include "mali_timeline_sync_fence.h" -+ -+int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs) -+{ -+ u32 val; -+ mali_timeline_id timeline; -+ mali_timeline_point point; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (0 != get_user(val, &uargs->timeline)) return -EFAULT; -+ -+ if (MALI_UK_TIMELINE_MAX <= val) { -+ return -EINVAL; -+ } -+ -+ timeline = (mali_timeline_id)val; -+ -+ point = mali_timeline_system_get_latest_point(session->timeline_system, timeline); -+ -+ if (0 != put_user(point, &uargs->point)) return -EFAULT; -+ -+ return 0; -+} -+ -+int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs) -+{ -+ u32 timeout, status; -+ mali_bool ret; -+ _mali_uk_fence_t uk_fence; -+ struct mali_timeline_fence fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; -+ if (0 != get_user(timeout, &uargs->timeout)) return -EFAULT; -+ -+ mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); -+ -+ ret = mali_timeline_fence_wait(session->timeline_system, &fence, timeout); -+ status = (MALI_TRUE == ret ? 1 : 0); -+ -+ if (0 != put_user(status, &uargs->status)) return -EFAULT; -+ -+ return 0; -+} -+ -+int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs) -+{ -+ s32 sync_fd = -1; -+ _mali_uk_fence_t uk_fence; -+ struct mali_timeline_fence fence; -+ -+ MALI_DEBUG_ASSERT_POINTER(session); -+ -+ if (0 != copy_from_user(&uk_fence, &uargs->fence, sizeof(_mali_uk_fence_t))) return -EFAULT; -+ mali_timeline_fence_copy_uk_fence(&fence, &uk_fence); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ sync_fd = mali_timeline_sync_fence_create(session->timeline_system, &fence); -+#else -+ sync_fd = -1; -+#endif /* defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) */ -+ -+ if (0 != put_user(sync_fd, &uargs->sync_fd)) return -EFAULT; -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c -new file mode 100755 -index 000000000..52519d1f9 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_vsync.c -@@ -0,0 +1,39 @@ -+/* -+ * Copyright (C) 2011-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+#include /* file system operations */ -+#include /* user space access */ -+ -+#include "mali_ukk.h" -+#include "mali_osk.h" -+#include "mali_kernel_common.h" -+#include "mali_session.h" -+#include "mali_ukk_wrappers.h" -+ -+ -+int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs) -+{ -+ _mali_uk_vsync_event_report_s kargs; -+ _mali_osk_errcode_t err; -+ -+ MALI_CHECK_NON_NULL(uargs, -EINVAL); -+ -+ if (0 != copy_from_user(&kargs, uargs, sizeof(_mali_uk_vsync_event_report_s))) { -+ return -EFAULT; -+ } -+ -+ kargs.ctx = (uintptr_t)session_data; -+ err = _mali_ukk_vsync_event_report(&kargs); -+ if (_MALI_OSK_ERR_OK != err) { -+ return map_errcode(err); -+ } -+ -+ return 0; -+} -+ -diff --git a/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h -new file mode 100755 -index 000000000..1add628fe ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/linux/mali_ukk_wrappers.h -@@ -0,0 +1,82 @@ -+/* -+ * Copyright (C) 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_ukk_wrappers.h -+ * Defines the wrapper functions for each user-kernel function -+ */ -+ -+#ifndef __MALI_UKK_WRAPPERS_H__ -+#define __MALI_UKK_WRAPPERS_H__ -+ -+#include "mali_uk_types.h" -+#include "mali_osk.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+int wait_for_notification_wrapper(struct mali_session_data *session_data, _mali_uk_wait_for_notification_s __user *uargs); -+int get_api_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_s __user *uargs); -+int get_api_version_v2_wrapper(struct mali_session_data *session_data, _mali_uk_get_api_version_v2_s __user *uargs); -+int get_user_settings_wrapper(struct mali_session_data *session_data, _mali_uk_get_user_settings_s __user *uargs); -+int post_notification_wrapper(struct mali_session_data *session_data, _mali_uk_post_notification_s __user *uargs); -+int request_high_priority_wrapper(struct mali_session_data *session_data, _mali_uk_request_high_priority_s __user *uargs); -+int pending_submit_wrapper(struct mali_session_data *session_data, _mali_uk_pending_submit_s __user *uargs); -+ -+/* rk_ext : 从对 r5p0-01rel0 集æˆå¼€å§‹, ä¸å†ä½¿ç”¨. */ -+#if 0 -+int get_mali_version_in_rk30_wrapper(struct mali_session_data *session_data, _mali_uk_get_mali_version_in_rk30_s __user *uargs); -+#else -+int get_rk_ko_version_wrapper(struct mali_session_data *session_data, _mali_rk_ko_version_s __user *uargs); -+#endif -+ -+int mem_alloc_wrapper(struct mali_session_data *session_data, _mali_uk_alloc_mem_s __user *uargs); -+int mem_free_wrapper(struct mali_session_data *session_data, _mali_uk_free_mem_s __user *uargs); -+int mem_bind_wrapper(struct mali_session_data *session_data, _mali_uk_bind_mem_s __user *uargs); -+int mem_unbind_wrapper(struct mali_session_data *session_data, _mali_uk_unbind_mem_s __user *uargs); -+int mem_cow_wrapper(struct mali_session_data *session_data, _mali_uk_cow_mem_s __user *uargs); -+int mem_cow_modify_range_wrapper(struct mali_session_data *session_data, _mali_uk_cow_modify_range_s __user *uargs); -+int mem_resize_mem_wrapper(struct mali_session_data *session_data, _mali_uk_mem_resize_s __user *uargs); -+int mem_write_safe_wrapper(struct mali_session_data *session_data, _mali_uk_mem_write_safe_s __user *uargs); -+int mem_query_mmu_page_table_dump_size_wrapper(struct mali_session_data *session_data, _mali_uk_query_mmu_page_table_dump_size_s __user *uargs); -+int mem_dump_mmu_page_table_wrapper(struct mali_session_data *session_data, _mali_uk_dump_mmu_page_table_s __user *uargs); -+int mem_usage_get_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_memory_usage_get_s __user *uargs); -+ -+int timeline_get_latest_point_wrapper(struct mali_session_data *session, _mali_uk_timeline_get_latest_point_s __user *uargs); -+int timeline_wait_wrapper(struct mali_session_data *session, _mali_uk_timeline_wait_s __user *uargs); -+int timeline_create_sync_fence_wrapper(struct mali_session_data *session, _mali_uk_timeline_create_sync_fence_s __user *uargs); -+int soft_job_start_wrapper(struct mali_session_data *session, _mali_uk_soft_job_start_s __user *uargs); -+int soft_job_signal_wrapper(struct mali_session_data *session, _mali_uk_soft_job_signal_s __user *uargs); -+int pp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_start_job_s __user *uargs); -+int pp_and_gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_pp_and_gp_start_job_s __user *uargs); -+int pp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_number_of_cores_s __user *uargs); -+int pp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_pp_core_version_s __user *uargs); -+int pp_disable_wb_wrapper(struct mali_session_data *session_data, _mali_uk_pp_disable_wb_s __user *uargs); -+int gp_start_job_wrapper(struct mali_session_data *session_data, _mali_uk_gp_start_job_s __user *uargs); -+int gp_get_number_of_cores_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_number_of_cores_s __user *uargs); -+int gp_get_core_version_wrapper(struct mali_session_data *session_data, _mali_uk_get_gp_core_version_s __user *uargs); -+int gp_suspend_response_wrapper(struct mali_session_data *session_data, _mali_uk_gp_suspend_response_s __user *uargs); -+ -+int profiling_add_event_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_add_event_s __user *uargs); -+int profiling_report_sw_counters_wrapper(struct mali_session_data *session_data, _mali_uk_sw_counters_report_s __user *uargs); -+int profiling_get_stream_fd_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_stream_fd_get_s __user *uargs); -+int profiling_control_set_wrapper(struct mali_session_data *session_data, _mali_uk_profiling_control_set_s __user *uargs); -+ -+int vsync_event_report_wrapper(struct mali_session_data *session_data, _mali_uk_vsync_event_report_s __user *uargs); -+ -+ -+int map_errcode(_mali_osk_errcode_t err); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __MALI_UKK_WRAPPERS_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c -new file mode 100755 -index 000000000..fc7017bbf ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm.c -@@ -0,0 +1,629 @@ -+/* -+ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file mali_platform.c -+ * Platform specific Mali driver functions for: -+ * - Realview Versatile platforms with ARM11 Mpcore and virtex 5. -+ * - Versatile Express platforms with ARM Cortex-A9 and virtex 6. -+ */ -+#include -+#include -+#include -+#include "mali_kernel_linux.h" -+#ifdef CONFIG_PM_RUNTIME -+#include -+#endif -+#include -+#include -+#include "mali_kernel_common.h" -+#include -+#include -+ -+#include "arm_core_scaling.h" -+#include "mali_executor.h" -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+#include -+#include -+#endif -+ -+static int mali_core_scaling_enable = 0; -+ -+void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data); -+static u32 mali_read_phys(u32 phys_addr); -+#if defined(CONFIG_ARCH_REALVIEW) -+static void mali_write_phys(u32 phys_addr, u32 value); -+#endif -+ -+#if defined(CONFIG_ARCH_VEXPRESS) && defined(CONFIG_ARM64) -+ -+#define SECURE_MODE_CONTROL_HANDLER 0x6F02006C -+void *secure_mode_mapped_addr = NULL; -+/** -+ * Reset GPU and enable/disable Mali secure mode. -+ * @Return value: -+ * 0: success -+ * non-0: failure. -+ */ -+ -+static int mali_gpu_reset_and_secure_mode_enable_juno(void) -+{ -+ u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; -+ MALI_DEBUG_ASSERT(NULL != secure_mode_mapped_addr); -+ -+ iowrite32(1, ((u8 *)secure_mode_mapped_addr) + phys_offset); -+ -+ if (1 == (u32)ioread32(((u8 *)secure_mode_mapped_addr) + phys_offset)) { -+ MALI_DEBUG_PRINT(3, ("Mali reset GPU and enable secured mode successfully! \n")); -+ return 0; -+ } -+ -+ MALI_PRINT_ERROR(("Failed to reset GPU and enable Mali secured mode !!! \n")); -+ -+ return -1; -+ -+} -+ -+static int mali_gpu_reset_and_secure_mode_disable_juno(void) -+{ -+ u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; -+ MALI_DEBUG_ASSERT(NULL != secure_mode_mapped_addr); -+ -+ iowrite32(0, ((u8 *)secure_mode_mapped_addr) + phys_offset); -+ -+ if (0 == (u32)ioread32(((u8 *)secure_mode_mapped_addr) + phys_offset)) { -+ MALI_DEBUG_PRINT(3, ("Mali reset GPU and disable secured mode successfully! \n")); -+ return 0; -+ } -+ -+ MALI_PRINT_ERROR(("Failed to reset GPU and disable mali secured mode !!! \n")); -+ return -1; -+} -+ -+static int mali_secure_mode_init_juno(void) -+{ -+ u32 phys_addr_page = SECURE_MODE_CONTROL_HANDLER & 0xFFFFE000; -+ u32 phys_offset = SECURE_MODE_CONTROL_HANDLER & 0x00001FFF; -+ u32 map_size = phys_offset + sizeof(u32); -+ -+ MALI_DEBUG_ASSERT(NULL == secure_mode_mapped_addr); -+ -+ secure_mode_mapped_addr = ioremap(phys_addr_page, map_size); -+ if (NULL != secure_mode_mapped_addr) { -+ return mali_gpu_reset_and_secure_mode_disable_juno(); -+ } -+ MALI_DEBUG_PRINT(2, ("Failed to ioremap for Mali secured mode! \n")); -+ return -1; -+} -+ -+static void mali_secure_mode_deinit_juno(void) -+{ -+ if (NULL != secure_mode_mapped_addr) { -+ mali_gpu_reset_and_secure_mode_disable_juno(); -+ iounmap(secure_mode_mapped_addr); -+ secure_mode_mapped_addr = NULL; -+ } -+} -+#endif -+ -+#ifndef CONFIG_MALI_DT -+static void mali_platform_device_release(struct device *device); -+ -+#if defined(CONFIG_ARCH_VEXPRESS) -+ -+#if defined(CONFIG_ARM64) -+/* Juno + Mali-450 MP6 in V7 FPGA */ -+static struct resource mali_gpu_resources_m450_mp6[] = { -+ MALI_GPU_RESOURCES_MALI450_MP6_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) -+}; -+ -+static struct resource mali_gpu_resources_m470_mp4[] = { -+ MALI_GPU_RESOURCES_MALI470_MP4_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200) -+}; -+ -+static struct resource mali_gpu_resources_m470_mp3[] = { -+ MALI_GPU_RESOURCES_MALI470_MP3_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200, 200, 200) -+}; -+ -+static struct resource mali_gpu_resources_m470_mp2[] = { -+ MALI_GPU_RESOURCES_MALI470_MP2_PMU(0x6F040000, 200, 200, 200, 200, 200, 200, 200) -+}; -+ -+static struct resource mali_gpu_resources_m470_mp1[] = { -+ MALI_GPU_RESOURCES_MALI470_MP1_PMU(0x6F040000, 200, 200, 200, 200, 200) -+}; -+ -+#else -+static struct resource mali_gpu_resources_m450_mp8[] = { -+ MALI_GPU_RESOURCES_MALI450_MP8_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) -+}; -+ -+static struct resource mali_gpu_resources_m450_mp6[] = { -+ MALI_GPU_RESOURCES_MALI450_MP6_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) -+}; -+ -+static struct resource mali_gpu_resources_m450_mp4[] = { -+ MALI_GPU_RESOURCES_MALI450_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) -+}; -+ -+static struct resource mali_gpu_resources_m470_mp4[] = { -+ MALI_GPU_RESOURCES_MALI470_MP4_PMU(0xFC040000, -1, 70, 70, 70, 70, 70, 70, 70, 70, 70, 68) -+}; -+#endif /* CONFIG_ARM64 */ -+ -+#elif defined(CONFIG_ARCH_REALVIEW) -+ -+static struct resource mali_gpu_resources_m300[] = { -+ MALI_GPU_RESOURCES_MALI300_PMU(0xC0000000, -1, -1, -1, -1) -+}; -+ -+static struct resource mali_gpu_resources_m400_mp1[] = { -+ MALI_GPU_RESOURCES_MALI400_MP1_PMU(0xC0000000, -1, -1, -1, -1) -+}; -+ -+static struct resource mali_gpu_resources_m400_mp2[] = { -+ MALI_GPU_RESOURCES_MALI400_MP2_PMU(0xC0000000, -1, -1, -1, -1, -1, -1) -+}; -+ -+#endif -+#endif -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ -+#define FALLBACK_STATIC_TEMPERATURE 55000 -+ -+static struct thermal_zone_device *gpu_tz; -+ -+/* Calculate gpu static power example for reference */ -+static unsigned long arm_model_static_power(struct devfreq *devfreq, -+ unsigned long voltage) -+{ -+ int temperature, temp; -+ int temp_squared, temp_cubed, temp_scaling_factor; -+ const unsigned long coefficient = (410UL << 20) / (729000000UL >> 10); -+ const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; -+ unsigned long static_power; -+ -+ if (gpu_tz) { -+ int ret; -+ -+ ret = gpu_tz->ops->get_temp(gpu_tz, &temperature); -+ if (ret) { -+ MALI_DEBUG_PRINT(2, ("Error reading temperature for gpu thermal zone: %d\n", ret)); -+ temperature = FALLBACK_STATIC_TEMPERATURE; -+ } -+ } else { -+ temperature = FALLBACK_STATIC_TEMPERATURE; -+ } -+ -+ /* Calculate the temperature scaling factor. To be applied to the -+ * voltage scaled power. -+ */ -+ temp = temperature / 1000; -+ temp_squared = temp * temp; -+ temp_cubed = temp_squared * temp; -+ temp_scaling_factor = -+ (2 * temp_cubed) -+ - (80 * temp_squared) -+ + (4700 * temp) -+ + 32000; -+ -+ static_power = (((coefficient * voltage_cubed) >> 20) -+ * temp_scaling_factor) -+ / 1000000; -+ -+ return static_power; -+} -+ -+/* Calculate gpu dynamic power example for reference */ -+static unsigned long arm_model_dynamic_power(struct devfreq *devfreq, -+ unsigned long freq, -+ unsigned long voltage) -+{ -+ /* The inputs: freq (f) is in Hz, and voltage (v) in mV. -+ * The coefficient (c) is in mW/(MHz mV mV). -+ * -+ * This function calculates the dynamic power after this formula: -+ * Pdyn (mW) = c (mW/(MHz*mV*mV)) * v (mV) * v (mV) * f (MHz) -+ */ -+ const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */ -+ const unsigned long f_mhz = freq / 1000000; /* MHz */ -+ const unsigned long coefficient = 3600; /* mW/(MHz*mV*mV) */ -+ unsigned long dynamic_power; -+ -+ dynamic_power = (coefficient * v2 * f_mhz) / 1000000; /* mW */ -+ -+ return dynamic_power; -+} -+ -+struct devfreq_cooling_power arm_cooling_ops = { -+ .get_static_power = arm_model_static_power, -+ .get_dynamic_power = arm_model_dynamic_power, -+}; -+#endif -+ -+static struct mali_gpu_device_data mali_gpu_data = { -+#ifndef CONFIG_MALI_DT -+ .pmu_switch_delay = 0xFF, /* do not have to be this high on FPGA, but it is good for testing to have a delay */ -+#if defined(CONFIG_ARCH_VEXPRESS) -+ .shared_mem_size = 256 * 1024 * 1024, /* 256MB */ -+#endif -+#endif -+ .max_job_runtime = 60000, /* 60 seconds */ -+ -+#if defined(CONFIG_ARCH_REALVIEW) -+ .dedicated_mem_start = 0x80000000, /* Physical start address (use 0xD0000000 for old indirect setup) */ -+ .dedicated_mem_size = 0x10000000, /* 256MB */ -+#endif -+#if defined(CONFIG_ARM64) -+ /* Some framebuffer drivers get the framebuffer dynamically, such as through GEM, -+ * in which the memory resource can't be predicted in advance. -+ */ -+ .fb_start = 0x0, -+ .fb_size = 0xFFFFF000, -+#else -+ .fb_start = 0xe0000000, -+ .fb_size = 0x01000000, -+#endif -+ .control_interval = 1000, /* 1000ms */ -+ .utilization_callback = mali_gpu_utilization_callback, -+ .get_clock_info = NULL, -+ .get_freq = NULL, -+ .set_freq = NULL, -+#if defined(CONFIG_ARCH_VEXPRESS) && defined(CONFIG_ARM64) -+ .secure_mode_init = mali_secure_mode_init_juno, -+ .secure_mode_deinit = mali_secure_mode_deinit_juno, -+ .gpu_reset_and_secure_mode_enable = mali_gpu_reset_and_secure_mode_enable_juno, -+ .gpu_reset_and_secure_mode_disable = mali_gpu_reset_and_secure_mode_disable_juno, -+#else -+ .secure_mode_init = NULL, -+ .secure_mode_deinit = NULL, -+ .gpu_reset_and_secure_mode_enable = NULL, -+ .gpu_reset_and_secure_mode_disable = NULL, -+#endif -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ .gpu_cooling_ops = &arm_cooling_ops, -+#endif -+}; -+ -+#ifndef CONFIG_MALI_DT -+static struct platform_device mali_gpu_device = { -+ .name = MALI_GPU_NAME_UTGARD, -+ .id = 0, -+ .dev.release = mali_platform_device_release, -+ .dev.dma_mask = &mali_gpu_device.dev.coherent_dma_mask, -+ .dev.coherent_dma_mask = DMA_BIT_MASK(32), -+ -+ .dev.platform_data = &mali_gpu_data, -+}; -+ -+int mali_platform_device_register(void) -+{ -+ int err = -1; -+ int num_pp_cores = 0; -+#if defined(CONFIG_ARCH_REALVIEW) -+ u32 m400_gp_version; -+#endif -+ -+ MALI_DEBUG_PRINT(4, ("mali_platform_device_register() called\n")); -+ -+ /* Detect present Mali GPU and connect the correct resources to the device */ -+#if defined(CONFIG_ARCH_VEXPRESS) -+ -+#if defined(CONFIG_ARM64) -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) -+ mali_gpu_device.dev.archdata.dma_ops = &dummy_dma_ops; -+#else -+ mali_gpu_device.dev.archdata.dma_ops = dma_ops; -+#endif -+ if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); -+ num_pp_cores = 6; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); -+ mali_gpu_device.resource = mali_gpu_resources_m450_mp6; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); -+ num_pp_cores = 4; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); -+ mali_gpu_device.resource = mali_gpu_resources_m470_mp4; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); -+ num_pp_cores = 3; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp3); -+ mali_gpu_device.resource = mali_gpu_resources_m470_mp3; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); -+ num_pp_cores = 2; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp2); -+ mali_gpu_device.resource = mali_gpu_resources_m470_mp2; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); -+ num_pp_cores = 1; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp1); -+ mali_gpu_device.resource = mali_gpu_resources_m470_mp1; -+ } -+#else -+ if (mali_read_phys(0xFC000000) == 0x00000450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); -+ num_pp_cores = 8; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp8); -+ mali_gpu_device.resource = mali_gpu_resources_m450_mp8; -+ } else if (mali_read_phys(0xFC000000) == 0x40600450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); -+ num_pp_cores = 6; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp6); -+ mali_gpu_device.resource = mali_gpu_resources_m450_mp6; -+ } else if (mali_read_phys(0xFC000000) == 0x40400450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); -+ num_pp_cores = 4; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m450_mp4); -+ mali_gpu_device.resource = mali_gpu_resources_m450_mp4; -+ } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); -+ num_pp_cores = 4; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m470_mp4); -+ mali_gpu_device.resource = mali_gpu_resources_m470_mp4; -+ } -+#endif /* CONFIG_ARM64 */ -+ -+#elif defined(CONFIG_ARCH_REALVIEW) -+ -+ m400_gp_version = mali_read_phys(0xC000006C); -+ if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); -+ num_pp_cores = 1; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m300); -+ mali_gpu_device.resource = mali_gpu_resources_m300; -+ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ -+ } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { -+ u32 fpga_fw_version = mali_read_phys(0xC0010000); -+ if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { -+ /* Mali-400 MP1 r1p0 or r1p1 */ -+ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); -+ num_pp_cores = 1; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp1); -+ mali_gpu_device.resource = mali_gpu_resources_m400_mp1; -+ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ -+ } else if (fpga_fw_version == 0x130C000F) { -+ /* Mali-400 MP2 r1p1 */ -+ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); -+ num_pp_cores = 2; -+ mali_gpu_device.num_resources = ARRAY_SIZE(mali_gpu_resources_m400_mp2); -+ mali_gpu_device.resource = mali_gpu_resources_m400_mp2; -+ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ -+ } -+ } -+ -+#endif -+ /* Register the platform device */ -+ err = platform_device_register(&mali_gpu_device); -+ if (0 == err) { -+#ifdef CONFIG_PM_RUNTIME -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) -+ pm_runtime_set_autosuspend_delay(&(mali_gpu_device.dev), 1000); -+ pm_runtime_use_autosuspend(&(mali_gpu_device.dev)); -+#endif -+ pm_runtime_enable(&(mali_gpu_device.dev)); -+#endif -+ MALI_DEBUG_ASSERT(0 < num_pp_cores); -+ mali_core_scaling_init(num_pp_cores); -+ -+ return 0; -+ } -+ -+ return err; -+} -+ -+void mali_platform_device_unregister(void) -+{ -+ MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); -+ -+ mali_core_scaling_term(); -+#ifdef CONFIG_PM_RUNTIME -+ pm_runtime_disable(&(mali_gpu_device.dev)); -+#endif -+ platform_device_unregister(&mali_gpu_device); -+ -+ platform_device_put(&mali_gpu_device); -+ -+#if defined(CONFIG_ARCH_REALVIEW) -+ mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ -+#endif -+} -+ -+static void mali_platform_device_release(struct device *device) -+{ -+ MALI_DEBUG_PRINT(4, ("mali_platform_device_release() called\n")); -+} -+ -+#else /* CONFIG_MALI_DT */ -+int mali_platform_device_init(struct platform_device *device) -+{ -+ int num_pp_cores = 0; -+ int err = -1; -+#if defined(CONFIG_ARCH_REALVIEW) -+ u32 m400_gp_version; -+#endif -+ -+ /* Detect present Mali GPU and connect the correct resources to the device */ -+#if defined(CONFIG_ARCH_VEXPRESS) -+ -+#if defined(CONFIG_ARM64) -+ if ((mali_read_phys(0x6F000000) & 0x00600450) == 0x00600450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP6 device\n")); -+ num_pp_cores = 6; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00400430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); -+ num_pp_cores = 4; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00300430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP3 device\n")); -+ num_pp_cores = 3; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00200430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP2 device\n")); -+ num_pp_cores = 2; -+ } else if ((mali_read_phys(0x6F000000) & 0x00F00430) == 0x00100430) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP1 device\n")); -+ num_pp_cores = 1; -+ } -+#else -+ if (mali_read_phys(0xFC000000) == 0x00000450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP8 device\n")); -+ num_pp_cores = 8; -+ } else if (mali_read_phys(0xFC000000) == 0x40400450) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-450 MP4 device\n")); -+ num_pp_cores = 4; -+ } else if (mali_read_phys(0xFC000000) == 0xFFFFFFFF) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-470 MP4 device\n")); -+ num_pp_cores = 4; -+ } -+#endif -+ -+#elif defined(CONFIG_ARCH_REALVIEW) -+ -+ m400_gp_version = mali_read_phys(0xC000006C); -+ if ((m400_gp_version & 0xFFFF0000) == 0x0C070000) { -+ MALI_DEBUG_PRINT(4, ("Registering Mali-300 device\n")); -+ num_pp_cores = 1; -+ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ -+ } else if ((m400_gp_version & 0xFFFF0000) == 0x0B070000) { -+ u32 fpga_fw_version = mali_read_phys(0xC0010000); -+ if (fpga_fw_version == 0x130C008F || fpga_fw_version == 0x110C008F) { -+ /* Mali-400 MP1 r1p0 or r1p1 */ -+ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP1 device\n")); -+ num_pp_cores = 1; -+ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ -+ } else if (fpga_fw_version == 0x130C000F) { -+ /* Mali-400 MP2 r1p1 */ -+ MALI_DEBUG_PRINT(4, ("Registering Mali-400 MP2 device\n")); -+ num_pp_cores = 2; -+ mali_write_phys(0xC0010020, 0xA); /* Enable direct memory mapping for FPGA */ -+ } -+ } -+#endif -+ -+ /* After kernel 3.15 device tree will default set dev -+ * related parameters in of_platform_device_create_pdata. -+ * But kernel changes from version to version, -+ * For example 3.10 didn't include device->dev.dma_mask parameter setting, -+ * if we didn't include here will cause dma_mapping error, -+ * but in kernel 3.15 it include device->dev.dma_mask parameter setting, -+ * so it's better to set must need paramter by DDK itself. -+ */ -+ if (!device->dev.dma_mask) -+ device->dev.dma_mask = &device->dev.coherent_dma_mask; -+ device->dev.archdata.dma_ops = dma_ops; -+ -+ err = platform_device_add_data(device, &mali_gpu_data, sizeof(mali_gpu_data)); -+ -+ if (0 == err) { -+#ifdef CONFIG_PM_RUNTIME -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) -+ pm_runtime_set_autosuspend_delay(&(device->dev), 1000); -+ pm_runtime_use_autosuspend(&(device->dev)); -+#endif -+ pm_runtime_enable(&(device->dev)); -+#endif -+ MALI_DEBUG_ASSERT(0 < num_pp_cores); -+ mali_core_scaling_init(num_pp_cores); -+ } -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ /* Some Socs didn't support the devfreq thermal for mali */ -+ if (of_machine_is_compatible("rockchip,rk3036")) -+ return 0; -+ -+ /* Get thermal zone */ -+ gpu_tz = thermal_zone_get_zone_by_name("soc_thermal"); -+ if (IS_ERR(gpu_tz)) { -+ MALI_DEBUG_PRINT(2, ("Error getting gpu thermal zone (%ld), not yet ready?\n", -+ PTR_ERR(gpu_tz))); -+ gpu_tz = NULL; -+ -+ err = -EPROBE_DEFER; -+ } -+#endif -+ -+ return err; -+} -+ -+int mali_platform_device_deinit(struct platform_device *device) -+{ -+ MALI_IGNORE(device); -+ -+ MALI_DEBUG_PRINT(4, ("mali_platform_device_deinit() called\n")); -+ -+ mali_core_scaling_term(); -+#ifdef CONFIG_PM_RUNTIME -+ pm_runtime_disable(&(device->dev)); -+#endif -+ -+#if defined(CONFIG_ARCH_REALVIEW) -+ mali_write_phys(0xC0010020, 0x9); /* Restore default (legacy) memory mapping */ -+#endif -+ -+ return 0; -+} -+ -+#endif /* CONFIG_MALI_DT */ -+ -+static u32 mali_read_phys(u32 phys_addr) -+{ -+ u32 phys_addr_page = phys_addr & 0xFFFFE000; -+ u32 phys_offset = phys_addr & 0x00001FFF; -+ u32 map_size = phys_offset + sizeof(u32); -+ u32 ret = 0xDEADBEEF; -+ void *mem_mapped = ioremap(phys_addr_page, map_size); -+ if (NULL != mem_mapped) { -+ ret = (u32)ioread32(((u8 *)mem_mapped) + phys_offset); -+ iounmap(mem_mapped); -+ } -+ -+ return ret; -+} -+ -+#if defined(CONFIG_ARCH_REALVIEW) -+static void mali_write_phys(u32 phys_addr, u32 value) -+{ -+ u32 phys_addr_page = phys_addr & 0xFFFFE000; -+ u32 phys_offset = phys_addr & 0x00001FFF; -+ u32 map_size = phys_offset + sizeof(u32); -+ void *mem_mapped = ioremap(phys_addr_page, map_size); -+ if (NULL != mem_mapped) { -+ iowrite32(value, ((u8 *)mem_mapped) + phys_offset); -+ iounmap(mem_mapped); -+ } -+} -+#endif -+ -+static int param_set_core_scaling(const char *val, const struct kernel_param *kp) -+{ -+ int ret = param_set_int(val, kp); -+ -+ if (1 == mali_core_scaling_enable) { -+ mali_core_scaling_sync(mali_executor_get_num_cores_enabled()); -+ } -+ return ret; -+} -+ -+static struct kernel_param_ops param_ops_core_scaling = { -+ .set = param_set_core_scaling, -+ .get = param_get_int, -+}; -+ -+module_param_cb(mali_core_scaling_enable, ¶m_ops_core_scaling, &mali_core_scaling_enable, 0644); -+MODULE_PARM_DESC(mali_core_scaling_enable, "1 means to enable core scaling policy, 0 means to disable core scaling policy"); -+ -+void mali_gpu_utilization_callback(struct mali_gpu_utilization_data *data) -+{ -+ if (1 == mali_core_scaling_enable) { -+ mali_core_scaling_update(data); -+ } -+} -diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c -new file mode 100755 -index 000000000..7a2fc8107 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.c -@@ -0,0 +1,122 @@ -+/* -+ * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file arm_core_scaling.c -+ * Example core scaling policy. -+ */ -+ -+#include "arm_core_scaling.h" -+ -+#include -+#include "mali_kernel_common.h" -+ -+#include -+ -+static int num_cores_total; -+static int num_cores_enabled; -+ -+static struct work_struct wq_work; -+ -+static void set_num_cores(struct work_struct *work) -+{ -+ int err = mali_perf_set_num_pp_cores(num_cores_enabled); -+ MALI_DEBUG_ASSERT(0 == err); -+ MALI_IGNORE(err); -+} -+ -+static void enable_one_core(void) -+{ -+ if (num_cores_enabled < num_cores_total) { -+ ++num_cores_enabled; -+ schedule_work(&wq_work); -+ MALI_DEBUG_PRINT(3, ("Core scaling: Enabling one more core\n")); -+ } -+ -+ MALI_DEBUG_ASSERT(1 <= num_cores_enabled); -+ MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); -+} -+ -+static void disable_one_core(void) -+{ -+ if (1 < num_cores_enabled) { -+ --num_cores_enabled; -+ schedule_work(&wq_work); -+ MALI_DEBUG_PRINT(3, ("Core scaling: Disabling one core\n")); -+ } -+ -+ MALI_DEBUG_ASSERT(1 <= num_cores_enabled); -+ MALI_DEBUG_ASSERT(num_cores_total >= num_cores_enabled); -+} -+ -+static void enable_max_num_cores(void) -+{ -+ if (num_cores_enabled < num_cores_total) { -+ num_cores_enabled = num_cores_total; -+ schedule_work(&wq_work); -+ MALI_DEBUG_PRINT(3, ("Core scaling: Enabling maximum number of cores\n")); -+ } -+ -+ MALI_DEBUG_ASSERT(num_cores_total == num_cores_enabled); -+} -+ -+void mali_core_scaling_init(int num_pp_cores) -+{ -+ INIT_WORK(&wq_work, set_num_cores); -+ -+ num_cores_total = num_pp_cores; -+ num_cores_enabled = num_pp_cores; -+ -+ /* NOTE: Mali is not fully initialized at this point. */ -+} -+ -+void mali_core_scaling_sync(int num_cores) -+{ -+ num_cores_enabled = num_cores; -+} -+ -+void mali_core_scaling_term(void) -+{ -+ flush_scheduled_work(); -+} -+ -+#define PERCENT_OF(percent, max) ((int) ((percent)*(max)/100.0 + 0.5)) -+ -+void mali_core_scaling_update(struct mali_gpu_utilization_data *data) -+{ -+ /* -+ * This function implements a very trivial PP core scaling algorithm. -+ * -+ * It is _NOT_ of production quality. -+ * The only intention behind this algorithm is to exercise and test the -+ * core scaling functionality of the driver. -+ * It is _NOT_ tuned for neither power saving nor performance! -+ * -+ * Other metrics than PP utilization need to be considered as well -+ * in order to make a good core scaling algorithm. -+ */ -+ -+ MALI_DEBUG_PRINT(3, ("Utilization: (%3d, %3d, %3d), cores enabled: %d/%d\n", data->utilization_gpu, data->utilization_gp, data->utilization_pp, num_cores_enabled, num_cores_total)); -+ -+ /* NOTE: this function is normally called directly from the utilization callback which is in -+ * timer context. */ -+ -+ if (PERCENT_OF(90, 256) < data->utilization_pp) { -+ enable_max_num_cores(); -+ } else if (PERCENT_OF(50, 256) < data->utilization_pp) { -+ enable_one_core(); -+ } else if (PERCENT_OF(40, 256) < data->utilization_pp) { -+ /* do nothing */ -+ } else if (PERCENT_OF(0, 256) < data->utilization_pp) { -+ disable_one_core(); -+ } else { -+ /* do nothing */ -+ } -+} -diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h -new file mode 100755 -index 000000000..8e0101830 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/arm/arm_core_scaling.h -@@ -0,0 +1,44 @@ -+/* -+ * Copyright (C) 2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file arm_core_scaling.h -+ * Example core scaling policy. -+ */ -+ -+#ifndef __ARM_CORE_SCALING_H__ -+#define __ARM_CORE_SCALING_H__ -+ -+struct mali_gpu_utilization_data; -+ -+/** -+ * Initialize core scaling policy. -+ * -+ * @note The core scaling policy will assume that all PP cores are on initially. -+ * -+ * @param num_pp_cores Total number of PP cores. -+ */ -+void mali_core_scaling_init(int num_pp_cores); -+ -+/** -+ * Terminate core scaling policy. -+ */ -+void mali_core_scaling_term(void); -+ -+/** -+ * Update core scaling policy with new utilization data. -+ * -+ * @param data Utilization data. -+ */ -+void mali_core_scaling_update(struct mali_gpu_utilization_data *data); -+ -+void mali_core_scaling_sync(int num_cores); -+ -+#endif /* __ARM_CORE_SCALING_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c b/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c -new file mode 100755 -index 000000000..e4e7ab8b2 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/arm/juno_opp.c -@@ -0,0 +1,127 @@ -+/* -+ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file juno_opp.c -+ * Example: Set up opp table -+ * Using ARM64 juno specific SCPI_PROTOCOL get frequence inform -+ * Customer need implement your own platform releated logic -+ */ -+#ifdef CONFIG_ARCH_VEXPRESS -+#ifdef CONFIG_MALI_DEVFREQ -+#ifdef CONFIG_ARM64 -+#ifdef CONFIG_ARM_SCPI_PROTOCOL -+#include -+#include -+#include -+#include -+#include -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) -+#include -+#else /* Linux >= 3.13 */ -+/* In 3.13 the OPP include header file, types, and functions were all -+ * renamed. Use the old filename for the include, and define the new names to -+ * the old, when an old kernel is detected. -+ */ -+#include -+#define dev_pm_opp_add opp_add -+#define dev_pm_opp_remove opp_remove -+#endif /* Linux >= 3.13 */ -+ -+#include "mali_kernel_common.h" -+ -+static int init_juno_opps_from_scpi(struct device *dev) -+{ -+ struct scpi_dvfs_info *sinfo; -+ struct scpi_ops *sops; -+ -+ int i; -+ -+ sops = get_scpi_ops(); -+ if (NULL == sops) { -+ MALI_DEBUG_PRINT(2, ("Mali didn't get any scpi ops \n")); -+ return -1; -+ } -+ -+ /* Hard coded for Juno. 2 is GPU domain */ -+ sinfo = sops->dvfs_get_info(2); -+ if (IS_ERR_OR_NULL(sinfo)) -+ return PTR_ERR(sinfo); -+ -+ for (i = 0; i < sinfo->count; i++) { -+ struct scpi_opp *e = &sinfo->opps[i]; -+ -+ MALI_DEBUG_PRINT(2, ("Mali OPP from SCPI: %u Hz @ %u mV\n", e->freq, e->m_volt)); -+ -+ dev_pm_opp_add(dev, e->freq, e->m_volt * 1000); -+ } -+ -+ return 0; -+} -+ -+int setup_opps(void) -+{ -+ struct device_node *np; -+ struct platform_device *pdev; -+ int err; -+ -+ np = of_find_node_by_name(NULL, "gpu"); -+ if (!np) { -+ pr_err("Failed to find DT entry for Mali\n"); -+ return -EFAULT; -+ } -+ -+ pdev = of_find_device_by_node(np); -+ if (!pdev) { -+ pr_err("Failed to find device for Mali\n"); -+ of_node_put(np); -+ return -EFAULT; -+ } -+ -+ err = init_juno_opps_from_scpi(&pdev->dev); -+ -+ of_node_put(np); -+ -+ return err; -+} -+ -+int term_opps(struct device *dev) -+{ -+ struct scpi_dvfs_info *sinfo; -+ struct scpi_ops *sops; -+ -+ int i; -+ -+ sops = get_scpi_ops(); -+ if (NULL == sops) { -+ MALI_DEBUG_PRINT(2, ("Mali didn't get any scpi ops \n")); -+ return -1; -+ } -+ -+ /* Hard coded for Juno. 2 is GPU domain */ -+ sinfo = sops->dvfs_get_info(2); -+ if (IS_ERR_OR_NULL(sinfo)) -+ return PTR_ERR(sinfo); -+ -+ for (i = 0; i < sinfo->count; i++) { -+ struct scpi_opp *e = &sinfo->opps[i]; -+ -+ MALI_DEBUG_PRINT(2, ("Mali Remove OPP: %u Hz \n", e->freq)); -+ -+ dev_pm_opp_remove(dev, e->freq); -+ } -+ -+ return 0; -+ -+} -+#endif -+#endif -+#endif -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h b/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h -new file mode 100755 -index 000000000..fe5e12241 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/rk/custom_log.h -@@ -0,0 +1,209 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* ---------------------------------------------------------------------------- -+ * File: custom_log.h -+ * -+ * Desc: ChenZhen å好的 log 输出的定制实现. -+ * -+ * -------------------------------------------------------------------- -+ * < 习语 å’Œ 缩略语 > : -+ * -+ * -------------------------------------------------------------------- -+ * Usage: -+ * -+ * Note: -+ * -+ * Author: ChenZhen -+ * -+ * ---------------------------------------------------------------------------- -+ * Version: -+ * v1.0 -+ * ---------------------------------------------------------------------------- -+ * Log: -+ ----Fri Nov 19 15:20:28 2010 v1.0 -+ * -+ * ---------------------------------------------------------------------------- -+ */ -+ -+#ifndef __CUSTOM_LOG_H__ -+#define __CUSTOM_LOG_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/* ----------------------------------------------------------------------------- -+ * Include Files -+ * ----------------------------------------------------------------------------- -+ */ -+#include -+#include -+ -+/* ----------------------------------------------------------------------------- -+ * Macros Definition -+ * ----------------------------------------------------------------------------- -+ */ -+ -+/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ -+/* #define ENABLE_DEBUG_LOG */ -+ -+/*----------------------------------------------------------------------------*/ -+ -+#ifdef ENABLE_VERBOSE_LOG -+/** Verbose log. */ -+#define V(fmt, args...) \ -+ pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+#else -+#define V(...) ((void)0) -+#endif -+ -+#ifdef ENABLE_DEBUG_LOG -+/** Debug log. */ -+#define D(fmt, args...) \ -+ pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+#else -+#define D(...) ((void)0) -+#endif -+ -+#define I(fmt, args...) \ -+ pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+ -+#define W(fmt, args...) \ -+ pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ -+ fmt "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+ -+#define E(fmt, args...) \ -+ pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+ -+/*-------------------------------------------------------*/ -+ -+/** 使用 D(), 以å进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ -+#define D_DEC(var) D(#var " = %d.", var) -+ -+#define E_DEC(var) E(#var " = %d.", var) -+ -+/** 使用 D(), 以å六进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ -+#define D_HEX(var) D(#var " = 0x%x.", var) -+ -+#define E_HEX(var) E(#var " = 0x%x.", var) -+ -+/** -+ * 使用 D(), 以å六进制的形å¼, -+ * 打å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. -+ */ -+#define D_PTR(ptr) D(#ptr " = %p.", ptr) -+ -+#define E_PTR(ptr) E(#ptr " = %p.", ptr) -+ -+/** 使用 D(), æ‰“å° char 字串. */ -+#define D_STR(p_str) \ -+do { \ -+ if (!p_str) { \ -+ D(#p_str " = NULL."); \ -+ else \ -+ D(#p_str " = '%s'.", p_str); \ -+} while (0) -+ -+#define E_STR(p_str) \ -+do { \ -+ if (!p_str) \ -+ E(#p_str " = NULL."); \ -+ else \ -+ E(#p_str " = '%s'.", p_str); \ -+} while (0) -+ -+#ifdef ENABLE_DEBUG_LOG -+/** -+ * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. -+ */ -+#define D_MEM(p_start, len) \ -+do { \ -+ int i = 0; \ -+ char *p = (char *)(p_start); \ -+ D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ -+ (p_start), \ -+ (len)); \ -+ pr_debug("\t\t"); \ -+ for (i = 0; i < (len); i++) \ -+ pr_debug("0x%02x, ", p[i]); \ -+ pr_debug("\n"); \ -+} while (0) -+#else -+#define D_MEM(...) ((void)0) -+#endif -+ -+/*-------------------------------------------------------*/ -+ -+/** -+ * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, -+ * å°†å˜é‡ 'ret_var' 设置 'err_code', -+ * log 输出对应的 Error Caution, -+ * 然åŽè·³è½¬ 'label' 指定的代ç å¤„执行. -+ * @param msg -+ * 纯字串形å¼çš„æ示信æ¯. -+ * @param ret_var -+ * 标识函数执行状æ€æˆ–者结果的å˜é‡, -+ * 将被设置具体的 Error Code. -+ * 通常是 'ret' or 'result'. -+ * @param err_code -+ * 表å¾ç‰¹å®š error 的常数标识, -+ * 通常是 å®çš„å½¢æ€. -+ * @param label -+ * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, -+ * 通常就是 'EXIT'. -+ * @param args... -+ * 对应 'msg_fmt' 实å‚中, -+ * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. -+ */ -+#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ -+do { \ -+ E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ -+ (err_code), \ -+ ## args); \ -+ (ret_var) = (err_code); \ -+ goto label; \ -+} while (0) -+ -+/* ----------------------------------------------------------------------------- -+ * Types and Structures Definition -+ * ----------------------------------------------------------------------------- -+ */ -+ -+/* ----------------------------------------------------------------------------- -+ * Global Functions' Prototype -+ * ----------------------------------------------------------------------------- -+ */ -+ -+/* ----------------------------------------------------------------------------- -+ * Inline Functions Implementation -+ * ----------------------------------------------------------------------------- -+ */ -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __CUSTOM_LOG_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/rk.c b/drivers/gpu/arm/mali400/mali/platform/rk/rk.c -new file mode 100755 -index 000000000..9a012fdf8 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/rk/rk.c -@@ -0,0 +1,676 @@ -+/* -+ * (C) COPYRIGHT RockChip Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ */ -+ -+/** -+ * @file rk.c -+ * implementation of platform_specific_code on rk platforms, such as rk3328h. -+ * -+ * mali_device_driver(MDD) includes 2 parts : -+ * .DP : platform_dependent_part : -+ * located in /mali/platform// -+ * .DP : common_part : -+ * common part implemented by ARM. -+ */ -+ -+#define ENABLE_DEBUG_LOG -+#include "custom_log.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#ifdef CONFIG_PM -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include "mali_kernel_common.h" -+#include "../../common/mali_osk_mali.h" -+ -+/*---------------------------------------------------------------------------*/ -+ -+u32 mali_group_error; -+ -+/*---------------------------------------------------------------------------*/ -+ -+#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) -+ -+/* -+ * rk_platform_context_of_mali_device. -+ */ -+struct rk_context { -+ /* mali device. */ -+ struct device *dev; -+ /* is the GPU powered on? */ -+ bool is_powered; -+ /* debug only, the period in ms to count gpu_utilisation. */ -+ unsigned int utilisation_period; -+}; -+ -+struct rk_context *s_rk_context; -+ -+/*---------------------------------------------------------------------------*/ -+ -+#ifdef CONFIG_MALI_DEVFREQ -+static ssize_t utilisation_period_show(struct device *dev, -+ struct device_attribute *attr, -+ char *buf) -+{ -+ struct rk_context *platform = s_rk_context; -+ ssize_t ret = 0; -+ -+ ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); -+ -+ return ret; -+} -+ -+static ssize_t utilisation_period_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, -+ size_t count) -+{ -+ struct rk_context *platform = s_rk_context; -+ int ret = 0; -+ -+ ret = kstrtouint(buf, 0, &platform->utilisation_period); -+ if (ret) { -+ E("invalid input period : %s.", buf); -+ return ret; -+ } -+ D("set utilisation_period to '%d'.", platform->utilisation_period); -+ -+ return count; -+} -+ -+static ssize_t utilisation_show(struct device *dev, -+ struct device_attribute *attr, -+ char *buf) -+{ -+ struct rk_context *platform = s_rk_context; -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ ssize_t ret = 0; -+ unsigned long period_in_us = platform->utilisation_period * 1000; -+ unsigned long total_time; -+ unsigned long busy_time; -+ unsigned long utilisation; -+ -+ mali_pm_reset_dvfs_utilisation(mdev); -+ usleep_range(period_in_us, period_in_us + 100); -+ mali_pm_get_dvfs_utilisation(mdev, &total_time, &busy_time); -+ -+ /* 'devfreq_dev_profile' instance registered to devfreq -+ * also uses mali_pm_reset_dvfs_utilisation() -+ * and mali_pm_get_dvfs_utilisation(). -+ * So, it's better to disable GPU DVFS before reading this node. -+ */ -+ D("total_time : %lu, busy_time : %lu.", total_time, busy_time); -+ -+ utilisation = busy_time / (total_time / 100); -+ ret += snprintf(buf, PAGE_SIZE, "%lu\n", utilisation); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR_RW(utilisation_period); -+static DEVICE_ATTR_RO(utilisation); -+#endif -+ -+static int rk_context_create_sysfs_files(struct device *dev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ int ret; -+ -+ ret = device_create_file(dev, &dev_attr_utilisation_period); -+ if (ret) { -+ E("fail to create sysfs file 'utilisation_period'."); -+ goto out; -+ } -+ -+ ret = device_create_file(dev, &dev_attr_utilisation); -+ if (ret) { -+ E("fail to create sysfs file 'utilisation'."); -+ goto remove_utilisation_period; -+ } -+ -+ return 0; -+ -+remove_utilisation_period: -+ device_remove_file(dev, &dev_attr_utilisation_period); -+out: -+ return ret; -+#else -+ return 0; -+#endif -+} -+ -+static void rk_context_remove_sysfs_files(struct device *dev) -+{ -+#ifdef CONFIG_MALI_DEVFREQ -+ device_remove_file(dev, &dev_attr_utilisation_period); -+ device_remove_file(dev, &dev_attr_utilisation); -+#endif -+} -+ -+/*---------------------------------------------------------------------------*/ -+ -+/* -+ * Init rk_platform_context of mali_device. -+ */ -+static int rk_context_init(struct platform_device *pdev) -+{ -+ int ret = 0; -+ struct device *dev = &pdev->dev; -+ struct rk_context *platform; /* platform_context */ -+ -+ platform = kzalloc(sizeof(*platform), GFP_KERNEL); -+ if (!platform) { -+ E("no mem."); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ platform->dev = dev; -+ platform->is_powered = false; -+ -+ platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; -+ -+ ret = rk_context_create_sysfs_files(dev); -+ if (ret) { -+ E("fail to create sysfs files, ret = %d", ret); -+ goto EXIT; -+ } -+ -+ s_rk_context = platform; -+ -+ pm_runtime_set_autosuspend_delay(dev, 1000); -+ pm_runtime_use_autosuspend(dev); -+ pm_runtime_enable(dev); -+ -+EXIT: -+ return ret; -+} -+ -+static void rk_context_deinit(struct platform_device *pdev) -+{ -+ struct device *dev = &pdev->dev; -+ struct rk_context *platform = s_rk_context; -+ -+ pm_runtime_disable(dev); -+ -+ s_rk_context = NULL; -+ -+ rk_context_remove_sysfs_files(dev); -+ -+ if (platform) { -+ platform->is_powered = false; -+ platform->dev = NULL; -+ kfree(platform); -+ } -+} -+ -+/*---------------------------------------------------------------------------*/ -+/* for devfreq cooling. */ -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ -+#define FALLBACK_STATIC_TEMPERATURE 55000 -+ -+static u32 dynamic_coefficient; -+static u32 static_coefficient; -+static s32 ts[4]; -+static struct thermal_zone_device *gpu_tz; -+ -+static int power_model_simple_init(struct platform_device *pdev) -+{ -+ struct device_node *power_model_node; -+ const char *tz_name; -+ u32 static_power, dynamic_power; -+ u32 voltage, voltage_squared, voltage_cubed, frequency; -+ -+ power_model_node = of_get_child_by_name(pdev->dev.of_node, -+ "power_model"); -+ if (!power_model_node) { -+ dev_err(&pdev->dev, "could not find power_model node\n"); -+ return -ENODEV; -+ } -+ if (!of_device_is_compatible(power_model_node, -+ "arm,mali-simple-power-model")) { -+ dev_err(&pdev->dev, "power_model incompatible with simple power model\n"); -+ return -ENODEV; -+ } -+ -+ if (of_property_read_string(power_model_node, "thermal-zone", -+ &tz_name)) { -+ dev_err(&pdev->dev, "ts in power_model not available\n"); -+ return -EINVAL; -+ } -+ -+ gpu_tz = thermal_zone_get_zone_by_name(tz_name); -+ if (IS_ERR(gpu_tz)) { -+ pr_warn_ratelimited("Error getting gpu thermal zone '%s'(%ld), not yet ready?\n", -+ tz_name, -+ PTR_ERR(gpu_tz)); -+ gpu_tz = NULL; -+ } -+ -+ if (of_property_read_u32(power_model_node, "static-power", -+ &static_power)) { -+ dev_err(&pdev->dev, "static-power in power_model not available\n"); -+ return -EINVAL; -+ } -+ if (of_property_read_u32(power_model_node, "dynamic-power", -+ &dynamic_power)) { -+ dev_err(&pdev->dev, "dynamic-power in power_model not available\n"); -+ return -EINVAL; -+ } -+ if (of_property_read_u32(power_model_node, "voltage", -+ &voltage)) { -+ dev_err(&pdev->dev, "voltage in power_model not available\n"); -+ return -EINVAL; -+ } -+ if (of_property_read_u32(power_model_node, "frequency", -+ &frequency)) { -+ dev_err(&pdev->dev, "frequency in power_model not available\n"); -+ return -EINVAL; -+ } -+ voltage_squared = (voltage * voltage) / 1000; -+ voltage_cubed = voltage * voltage * voltage; -+ static_coefficient = (static_power << 20) / (voltage_cubed >> 10); -+ dynamic_coefficient = (((dynamic_power * 1000) / voltage_squared) -+ * 1000) / frequency; -+ -+ if (of_property_read_u32_array(power_model_node, "ts", (u32 *)ts, 4)) { -+ dev_err(&pdev->dev, "ts in power_model not available\n"); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+/* Calculate gpu static power example for reference */ -+static unsigned long rk_model_static_power(struct devfreq *devfreq, -+ unsigned long voltage) -+{ -+ int temperature, temp; -+ int temp_squared, temp_cubed, temp_scaling_factor; -+ const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10; -+ unsigned long static_power; -+ -+ if (gpu_tz) { -+ int ret; -+ -+ ret = gpu_tz->ops->get_temp(gpu_tz, &temperature); -+ if (ret) { -+ MALI_DEBUG_PRINT(2, ("fail to read temp: %d\n", ret)); -+ temperature = FALLBACK_STATIC_TEMPERATURE; -+ } -+ } else { -+ temperature = FALLBACK_STATIC_TEMPERATURE; -+ } -+ -+ /* Calculate the temperature scaling factor. To be applied to the -+ * voltage scaled power. -+ */ -+ temp = temperature / 1000; -+ temp_squared = temp * temp; -+ temp_cubed = temp_squared * temp; -+ temp_scaling_factor = -+ (ts[3] * temp_cubed) -+ + (ts[2] * temp_squared) -+ + (ts[1] * temp) -+ + ts[0]; -+ -+ static_power = (((static_coefficient * voltage_cubed) >> 20) -+ * temp_scaling_factor) -+ / 1000000; -+ -+ return static_power; -+} -+ -+/* Calculate gpu dynamic power example for reference */ -+static unsigned long rk_model_dynamic_power(struct devfreq *devfreq, -+ unsigned long freq, -+ unsigned long voltage) -+{ -+ /* The inputs: freq (f) is in Hz, and voltage (v) in mV. -+ * The coefficient (c) is in mW/(MHz mV mV). -+ * -+ * This function calculates the dynamic power after this formula: -+ * Pdyn (mW) = c (mW/(MHz*mV*mV)) * v (mV) * v (mV) * f (MHz) -+ */ -+ const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */ -+ const unsigned long f_mhz = freq / 1000000; /* MHz */ -+ unsigned long dynamic_power; -+ -+ dynamic_power = (dynamic_coefficient * v2 * f_mhz) / 1000000; /* mW */ -+ -+ return dynamic_power; -+} -+ -+struct devfreq_cooling_power rk_cooling_ops = { -+ .get_static_power = rk_model_static_power, -+ .get_dynamic_power = rk_model_dynamic_power, -+}; -+#endif -+ -+/*---------------------------------------------------------------------------*/ -+ -+#ifdef CONFIG_PM -+ -+static int rk_platform_enable_clk_gpu(struct device *dev) -+{ -+ int ret = 0; -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_HAVE_CLK) -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ if (mdev->clock) -+ ret = clk_enable(mdev->clock); -+#endif -+ return ret; -+} -+ -+static void rk_platform_disable_clk_gpu(struct device *dev) -+{ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_HAVE_CLK) -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ if (mdev->clock) -+ clk_disable(mdev->clock); -+#endif -+} -+ -+static int rk_platform_enable_gpu_regulator(struct device *dev) -+{ -+ int ret = 0; -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_REGULATOR) -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ if (mdev->regulator) -+ ret = regulator_enable(mdev->regulator); -+#endif -+ return ret; -+} -+ -+static void rk_platform_disable_gpu_regulator(struct device *dev) -+{ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_REGULATOR) -+ struct mali_device *mdev = dev_get_drvdata(dev); -+ -+ if (mdev->regulator) -+ regulator_disable(mdev->regulator); -+#endif -+} -+ -+static int rk_platform_power_on_gpu(struct device *dev) -+{ -+ struct rk_context *platform = s_rk_context; -+ int ret = 0; -+ -+ if (!(platform->is_powered)) { -+ ret = rk_platform_enable_clk_gpu(dev); -+ if (ret) { -+ E("fail to enable clk_gpu, ret : %d.", ret); -+ goto fail_to_enable_clk; -+ } -+ -+ ret = rk_platform_enable_gpu_regulator(dev); -+ if (ret) { -+ E("fail to enable vdd_gpu, ret : %d.", ret); -+ goto fail_to_enable_regulator; -+ } -+ -+ platform->is_powered = true; -+ } -+ -+ return 0; -+ -+fail_to_enable_regulator: -+ rk_platform_disable_clk_gpu(dev); -+ -+fail_to_enable_clk: -+ return ret; -+} -+ -+static void rk_platform_power_off_gpu(struct device *dev) -+{ -+ struct rk_context *platform = s_rk_context; -+ -+ if (platform->is_powered) { -+ rk_platform_disable_clk_gpu(dev); -+ rk_platform_disable_gpu_regulator(dev); -+ -+ platform->is_powered = false; -+ } -+} -+ -+int rk_platform_init_opp_table(struct device *dev) -+{ -+ return rockchip_init_opp_table(dev, NULL, "gpu_leakage", "mali"); -+} -+ -+static int mali_runtime_suspend(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_runtime_suspend() called\n")); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->runtime_suspend) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->runtime_suspend(device); -+ } -+ -+ if (!ret) -+ rk_platform_power_off_gpu(device); -+ -+ return ret; -+} -+ -+static int mali_runtime_resume(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_runtime_resume() called\n")); -+ -+ rk_platform_power_on_gpu(device); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->runtime_resume) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->runtime_resume(device); -+ } -+ -+ return ret; -+} -+ -+static int mali_runtime_idle(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_runtime_idle() called\n")); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->runtime_idle) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->runtime_idle(device); -+ if (ret) -+ return ret; -+ } -+ -+ return 0; -+} -+#endif -+ -+static int mali_os_suspend(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_os_suspend() called\n")); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->suspend) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->suspend(device); -+ } -+ -+ if (!ret) -+ rk_platform_power_off_gpu(device); -+ -+ return ret; -+} -+ -+static int mali_os_resume(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_os_resume() called\n")); -+ -+ rk_platform_power_on_gpu(device); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->resume) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->resume(device); -+ } -+ -+ return ret; -+} -+ -+static int mali_os_freeze(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_os_freeze() called\n")); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->freeze) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->freeze(device); -+ } -+ -+ return ret; -+} -+ -+static int mali_os_thaw(struct device *device) -+{ -+ int ret = 0; -+ -+ MALI_DEBUG_PRINT(4, ("mali_os_thaw() called\n")); -+ -+ if (device->driver && -+ device->driver->pm && -+ device->driver->pm->thaw) { -+ /* Need to notify Mali driver about this event */ -+ ret = device->driver->pm->thaw(device); -+ } -+ -+ return ret; -+} -+ -+static const struct dev_pm_ops mali_gpu_device_type_pm_ops = { -+ .suspend = mali_os_suspend, -+ .resume = mali_os_resume, -+ .freeze = mali_os_freeze, -+ .thaw = mali_os_thaw, -+#ifdef CONFIG_PM -+ .runtime_suspend = mali_runtime_suspend, -+ .runtime_resume = mali_runtime_resume, -+ .runtime_idle = mali_runtime_idle, -+#endif -+}; -+ -+static const struct device_type mali_gpu_device_device_type = { -+ .pm = &mali_gpu_device_type_pm_ops, -+}; -+ -+/* -+ * platform_specific_data of platform_device of mali_gpu. -+ */ -+static const struct mali_gpu_device_data mali_gpu_data = { -+ .shared_mem_size = 1024 * 1024 * 1024, /* 1GB */ -+ .max_job_runtime = 60000, /* 60 seconds */ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ .gpu_cooling_ops = &rk_cooling_ops, -+#endif -+}; -+ -+static void mali_platform_device_add_config(struct platform_device *pdev) -+{ -+ pdev->name = MALI_GPU_NAME_UTGARD, -+ pdev->id = 0; -+ pdev->dev.type = &mali_gpu_device_device_type; -+ pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask, -+ pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); -+} -+ -+/*---------------------------------------------------------------------------*/ -+/* platform_device_functions called by common_part. */ -+ -+int mali_platform_device_init(struct platform_device *pdev) -+{ -+ int err = 0; -+ -+ mali_platform_device_add_config(pdev); -+ -+ D("to add platform_specific_data to platform_device_of_mali."); -+ err = platform_device_add_data(pdev, -+ &mali_gpu_data, -+ sizeof(mali_gpu_data)); -+ if (err) { -+ E("fail to add platform_specific_data. err : %d.", err); -+ goto add_data_failed; -+ } -+ -+ err = rk_context_init(pdev); -+ if (err) { -+ E("fail to init rk_context. err : %d.", err); -+ goto init_rk_context_failed; -+ } -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ if (of_machine_is_compatible("rockchip,rk3036")) -+ return 0; -+ -+ err = power_model_simple_init(pdev); -+ if (err) { -+ E("fail to init simple_power_model, err : %d.", err); -+ goto init_power_model_failed; -+ } -+#endif -+ -+ return 0; -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+init_power_model_failed: -+ rk_context_deinit(pdev); -+#endif -+init_rk_context_failed: -+add_data_failed: -+ return err; -+} -+ -+void mali_platform_device_deinit(struct platform_device *pdev) -+{ -+ MALI_DEBUG_PRINT(4, ("mali_platform_device_unregister() called\n")); -+ -+ rk_context_deinit(pdev); -+} -diff --git a/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h b/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h -new file mode 100755 -index 000000000..bd939350c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/platform/rk/rk_ext.h -@@ -0,0 +1,37 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* ---------------------------------------------------------------------------- -+ * File: rk_ext.h -+ * -+ * Desc: rk_ext_on_mali_ko 中的 通行定义等. -+ * -+ * Usage: -+ * -+ * Note: -+ * -+ * Author: ChenZhen -+ * -+ * Log: -+ * -+ * ---------------------------------------------------------------------------- -+ */ -+ -+#ifndef __RK_EXT_H__ -+#define __RK_EXT_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/*---------------------------------------------------------------------------*/ -+ -+/** version of rk_ext on mali_ko, aka. rk_ko_ver. */ -+#define RK_KO_VER (5) -+ -+/*---------------------------------------------------------------------------*/ -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __RK_EXT_H__ */ -+ -diff --git a/drivers/gpu/arm/mali400/mali/readme.txt b/drivers/gpu/arm/mali400/mali/readme.txt -new file mode 100755 -index 000000000..6785ac933 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/readme.txt -@@ -0,0 +1,28 @@ -+Building the Mali Device Driver for Linux -+----------------------------------------- -+ -+Build the Mali Device Driver for Linux by running the following make command: -+ -+KDIR= USING_UMP= BUILD= make -+ -+where -+ kdir_path: Path to your Linux Kernel directory -+ ump_option: 1 = Enable UMP support(*) -+ 0 = disable UMP support -+ build_option: debug = debug build of driver -+ release = release build of driver -+ -+(*) For newer Linux Kernels, the Module.symvers file for the UMP device driver -+ must be available. The UMP_SYMVERS_FILE variable in the Makefile should -+ point to this file. This file is generated when the UMP driver is built. -+ -+The result will be a mali.ko file, which can be loaded into the Linux kernel -+by using the insmod command. -+ -+Use of UMP is not recommended. The dma-buf API in the Linux kernel has -+replaced UMP. The Mali Device Driver will be built with dma-buf support if the -+kernel config includes enabled dma-buf. -+ -+The kernel needs to be provided with a platform_device struct for the Mali GPU -+device. See the mali_utgard.h header file for how to set up the Mali GPU -+resources. -diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h -new file mode 100755 -index 000000000..0345fb169 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/regs/mali_200_regs.h -@@ -0,0 +1,131 @@ -+/* -+ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef _MALI200_REGS_H_ -+#define _MALI200_REGS_H_ -+ -+/** -+ * Enum for management register addresses. -+ */ -+enum mali200_mgmt_reg { -+ MALI200_REG_ADDR_MGMT_VERSION = 0x1000, -+ MALI200_REG_ADDR_MGMT_CURRENT_REND_LIST_ADDR = 0x1004, -+ MALI200_REG_ADDR_MGMT_STATUS = 0x1008, -+ MALI200_REG_ADDR_MGMT_CTRL_MGMT = 0x100c, -+ -+ MALI200_REG_ADDR_MGMT_INT_RAWSTAT = 0x1020, -+ MALI200_REG_ADDR_MGMT_INT_CLEAR = 0x1024, -+ MALI200_REG_ADDR_MGMT_INT_MASK = 0x1028, -+ MALI200_REG_ADDR_MGMT_INT_STATUS = 0x102c, -+ -+ MALI200_REG_ADDR_MGMT_BUS_ERROR_STATUS = 0x1050, -+ -+ MALI200_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x1080, -+ MALI200_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x1084, -+ MALI200_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x1088, -+ MALI200_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x108c, -+ -+ MALI200_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x10a0, -+ MALI200_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x10a4, -+ MALI200_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x10ac, -+ -+ MALI200_REG_ADDR_MGMT_PERFMON_CONTR = 0x10b0, -+ MALI200_REG_ADDR_MGMT_PERFMON_BASE = 0x10b4, -+ -+ MALI200_REG_SIZEOF_REGISTER_BANK = 0x10f0 -+ -+}; -+ -+#define MALI200_REG_VAL_PERF_CNT_ENABLE 1 -+ -+enum mali200_mgmt_ctrl_mgmt { -+ MALI200_REG_VAL_CTRL_MGMT_STOP_BUS = (1 << 0), -+ MALI200_REG_VAL_CTRL_MGMT_FLUSH_CACHES = (1 << 3), -+ MALI200_REG_VAL_CTRL_MGMT_FORCE_RESET = (1 << 5), -+ MALI200_REG_VAL_CTRL_MGMT_START_RENDERING = (1 << 6), -+ MALI400PP_REG_VAL_CTRL_MGMT_SOFT_RESET = (1 << 7), /* Only valid for Mali-300 and later */ -+}; -+ -+enum mali200_mgmt_irq { -+ MALI200_REG_VAL_IRQ_END_OF_FRAME = (1 << 0), -+ MALI200_REG_VAL_IRQ_END_OF_TILE = (1 << 1), -+ MALI200_REG_VAL_IRQ_HANG = (1 << 2), -+ MALI200_REG_VAL_IRQ_FORCE_HANG = (1 << 3), -+ MALI200_REG_VAL_IRQ_BUS_ERROR = (1 << 4), -+ MALI200_REG_VAL_IRQ_BUS_STOP = (1 << 5), -+ MALI200_REG_VAL_IRQ_CNT_0_LIMIT = (1 << 6), -+ MALI200_REG_VAL_IRQ_CNT_1_LIMIT = (1 << 7), -+ MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR = (1 << 8), -+ MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND = (1 << 9), -+ MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW = (1 << 10), -+ MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW = (1 << 11), -+ MALI400PP_REG_VAL_IRQ_RESET_COMPLETED = (1 << 12), -+}; -+ -+#define MALI200_REG_VAL_IRQ_MASK_ALL ((enum mali200_mgmt_irq) (\ -+ MALI200_REG_VAL_IRQ_END_OF_FRAME |\ -+ MALI200_REG_VAL_IRQ_END_OF_TILE |\ -+ MALI200_REG_VAL_IRQ_HANG |\ -+ MALI200_REG_VAL_IRQ_FORCE_HANG |\ -+ MALI200_REG_VAL_IRQ_BUS_ERROR |\ -+ MALI200_REG_VAL_IRQ_BUS_STOP |\ -+ MALI200_REG_VAL_IRQ_CNT_0_LIMIT |\ -+ MALI200_REG_VAL_IRQ_CNT_1_LIMIT |\ -+ MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ -+ MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ -+ MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ -+ MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW |\ -+ MALI400PP_REG_VAL_IRQ_RESET_COMPLETED)) -+ -+#define MALI200_REG_VAL_IRQ_MASK_USED ((enum mali200_mgmt_irq) (\ -+ MALI200_REG_VAL_IRQ_END_OF_FRAME |\ -+ MALI200_REG_VAL_IRQ_FORCE_HANG |\ -+ MALI200_REG_VAL_IRQ_BUS_ERROR |\ -+ MALI200_REG_VAL_IRQ_WRITE_BOUNDARY_ERROR |\ -+ MALI400PP_REG_VAL_IRQ_INVALID_PLIST_COMMAND |\ -+ MALI400PP_REG_VAL_IRQ_CALL_STACK_UNDERFLOW |\ -+ MALI400PP_REG_VAL_IRQ_CALL_STACK_OVERFLOW)) -+ -+#define MALI200_REG_VAL_IRQ_MASK_NONE ((enum mali200_mgmt_irq)(0)) -+ -+enum mali200_mgmt_status { -+ MALI200_REG_VAL_STATUS_RENDERING_ACTIVE = (1 << 0), -+ MALI200_REG_VAL_STATUS_BUS_STOPPED = (1 << 4), -+}; -+ -+enum mali200_render_unit { -+ MALI200_REG_ADDR_FRAME = 0x0000, -+ MALI200_REG_ADDR_RSW = 0x0004, -+ MALI200_REG_ADDR_STACK = 0x0030, -+ MALI200_REG_ADDR_STACK_SIZE = 0x0034, -+ MALI200_REG_ADDR_ORIGIN_OFFSET_X = 0x0040 -+}; -+ -+enum mali200_wb_unit { -+ MALI200_REG_ADDR_WB0 = 0x0100, -+ MALI200_REG_ADDR_WB1 = 0x0200, -+ MALI200_REG_ADDR_WB2 = 0x0300 -+}; -+ -+enum mali200_wb_unit_regs { -+ MALI200_REG_ADDR_WB_SOURCE_SELECT = 0x0000, -+ MALI200_REG_ADDR_WB_SOURCE_ADDR = 0x0004, -+}; -+ -+/* This should be in the top 16 bit of the version register of Mali PP */ -+#define MALI200_PP_PRODUCT_ID 0xC807 -+#define MALI300_PP_PRODUCT_ID 0xCE07 -+#define MALI400_PP_PRODUCT_ID 0xCD07 -+#define MALI450_PP_PRODUCT_ID 0xCF07 -+#define MALI470_PP_PRODUCT_ID 0xCF08 -+ -+ -+ -+#endif /* _MALI200_REGS_H_ */ -diff --git a/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h -new file mode 100755 -index 000000000..7f8b58fd6 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/regs/mali_gp_regs.h -@@ -0,0 +1,172 @@ -+/* -+ * Copyright (C) 2010, 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef _MALIGP2_CONROL_REGS_H_ -+#define _MALIGP2_CONROL_REGS_H_ -+ -+/** -+ * These are the different geometry processor control registers. -+ * Their usage is to control and monitor the operation of the -+ * Vertex Shader and the Polygon List Builder in the geometry processor. -+ * Addresses are in 32-bit word relative sizes. -+ * @see [P0081] "Geometry Processor Data Structures" for details -+ */ -+ -+typedef enum { -+ MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR = 0x00, -+ MALIGP2_REG_ADDR_MGMT_VSCL_END_ADDR = 0x04, -+ MALIGP2_REG_ADDR_MGMT_PLBUCL_START_ADDR = 0x08, -+ MALIGP2_REG_ADDR_MGMT_PLBUCL_END_ADDR = 0x0c, -+ MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_START_ADDR = 0x10, -+ MALIGP2_REG_ADDR_MGMT_PLBU_ALLOC_END_ADDR = 0x14, -+ MALIGP2_REG_ADDR_MGMT_CMD = 0x20, -+ MALIGP2_REG_ADDR_MGMT_INT_RAWSTAT = 0x24, -+ MALIGP2_REG_ADDR_MGMT_INT_CLEAR = 0x28, -+ MALIGP2_REG_ADDR_MGMT_INT_MASK = 0x2C, -+ MALIGP2_REG_ADDR_MGMT_INT_STAT = 0x30, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_ENABLE = 0x3C, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_ENABLE = 0x40, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC = 0x44, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_SRC = 0x48, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_VALUE = 0x4C, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_1_VALUE = 0x50, -+ MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_LIMIT = 0x54, -+ MALIGP2_REG_ADDR_MGMT_STATUS = 0x68, -+ MALIGP2_REG_ADDR_MGMT_VERSION = 0x6C, -+ MALIGP2_REG_ADDR_MGMT_VSCL_START_ADDR_READ = 0x80, -+ MALIGP2_REG_ADDR_MGMT_PLBCL_START_ADDR_READ = 0x84, -+ MALIGP2_CONTR_AXI_BUS_ERROR_STAT = 0x94, -+ MALIGP2_REGISTER_ADDRESS_SPACE_SIZE = 0x98, -+} maligp_reg_addr_mgmt_addr; -+ -+#define MALIGP2_REG_VAL_PERF_CNT_ENABLE 1 -+ -+/** -+ * Commands to geometry processor. -+ * @see MALIGP2_CTRL_REG_CMD -+ */ -+typedef enum { -+ MALIGP2_REG_VAL_CMD_START_VS = (1 << 0), -+ MALIGP2_REG_VAL_CMD_START_PLBU = (1 << 1), -+ MALIGP2_REG_VAL_CMD_UPDATE_PLBU_ALLOC = (1 << 4), -+ MALIGP2_REG_VAL_CMD_RESET = (1 << 5), -+ MALIGP2_REG_VAL_CMD_FORCE_HANG = (1 << 6), -+ MALIGP2_REG_VAL_CMD_STOP_BUS = (1 << 9), -+ MALI400GP_REG_VAL_CMD_SOFT_RESET = (1 << 10), /* only valid for Mali-300 and later */ -+} mgp_contr_reg_val_cmd; -+ -+ -+/** @defgroup MALIGP2_IRQ -+ * Interrupt status of geometry processor. -+ * @see MALIGP2_CTRL_REG_INT_RAWSTAT, MALIGP2_REG_ADDR_MGMT_INT_CLEAR, -+ * MALIGP2_REG_ADDR_MGMT_INT_MASK, MALIGP2_REG_ADDR_MGMT_INT_STAT -+ * @{ -+ */ -+#define MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST (1 << 0) -+#define MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST (1 << 1) -+#define MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM (1 << 2) -+#define MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ (1 << 3) -+#define MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ (1 << 4) -+#define MALIGP2_REG_VAL_IRQ_HANG (1 << 5) -+#define MALIGP2_REG_VAL_IRQ_FORCE_HANG (1 << 6) -+#define MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT (1 << 7) -+#define MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT (1 << 8) -+#define MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR (1 << 9) -+#define MALIGP2_REG_VAL_IRQ_SYNC_ERROR (1 << 10) -+#define MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR (1 << 11) -+#define MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED (1 << 12) -+#define MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD (1 << 13) -+#define MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD (1 << 14) -+#define MALI400GP_REG_VAL_IRQ_RESET_COMPLETED (1 << 19) -+#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW (1 << 20) -+#define MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW (1 << 21) -+#define MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS (1 << 22) -+ -+/* Mask defining all IRQs in Mali GP */ -+#define MALIGP2_REG_VAL_IRQ_MASK_ALL \ -+ (\ -+ MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ -+ MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ -+ MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ -+ MALIGP2_REG_VAL_IRQ_VS_SEM_IRQ | \ -+ MALIGP2_REG_VAL_IRQ_PLBU_SEM_IRQ | \ -+ MALIGP2_REG_VAL_IRQ_HANG | \ -+ MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ -+ MALIGP2_REG_VAL_IRQ_PERF_CNT_0_LIMIT | \ -+ MALIGP2_REG_VAL_IRQ_PERF_CNT_1_LIMIT | \ -+ MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ -+ MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ -+ MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ -+ MALI400GP_REG_VAL_IRQ_AXI_BUS_STOPPED | \ -+ MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ -+ MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ -+ MALI400GP_REG_VAL_IRQ_RESET_COMPLETED | \ -+ MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ -+ MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ -+ MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) -+ -+/* Mask defining the IRQs in Mali GP which we use */ -+#define MALIGP2_REG_VAL_IRQ_MASK_USED \ -+ (\ -+ MALIGP2_REG_VAL_IRQ_VS_END_CMD_LST | \ -+ MALIGP2_REG_VAL_IRQ_PLBU_END_CMD_LST | \ -+ MALIGP2_REG_VAL_IRQ_PLBU_OUT_OF_MEM | \ -+ MALIGP2_REG_VAL_IRQ_FORCE_HANG | \ -+ MALIGP2_REG_VAL_IRQ_WRITE_BOUND_ERR | \ -+ MALIGP2_REG_VAL_IRQ_SYNC_ERROR | \ -+ MALIGP2_REG_VAL_IRQ_AXI_BUS_ERROR | \ -+ MALI400GP_REG_VAL_IRQ_VS_INVALID_CMD | \ -+ MALI400GP_REG_VAL_IRQ_PLB_INVALID_CMD | \ -+ MALI400GP_REG_VAL_IRQ_SEMAPHORE_UNDERFLOW | \ -+ MALI400GP_REG_VAL_IRQ_SEMAPHORE_OVERFLOW | \ -+ MALI400GP_REG_VAL_IRQ_PTR_ARRAY_OUT_OF_BOUNDS) -+ -+/* Mask defining non IRQs on MaliGP2*/ -+#define MALIGP2_REG_VAL_IRQ_MASK_NONE 0 -+ -+/** }@ defgroup MALIGP2_IRQ*/ -+ -+/** @defgroup MALIGP2_STATUS -+ * The different Status values to the geometry processor. -+ * @see MALIGP2_CTRL_REG_STATUS -+ * @{ -+ */ -+#define MALIGP2_REG_VAL_STATUS_VS_ACTIVE 0x0002 -+#define MALIGP2_REG_VAL_STATUS_BUS_STOPPED 0x0004 -+#define MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE 0x0008 -+#define MALIGP2_REG_VAL_STATUS_BUS_ERROR 0x0040 -+#define MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR 0x0100 -+/** }@ defgroup MALIGP2_STATUS*/ -+ -+#define MALIGP2_REG_VAL_STATUS_MASK_ACTIVE (\ -+ MALIGP2_REG_VAL_STATUS_VS_ACTIVE|\ -+ MALIGP2_REG_VAL_STATUS_PLBU_ACTIVE) -+ -+ -+#define MALIGP2_REG_VAL_STATUS_MASK_ERROR (\ -+ MALIGP2_REG_VAL_STATUS_BUS_ERROR |\ -+ MALIGP2_REG_VAL_STATUS_WRITE_BOUND_ERR ) -+ -+/* This should be in the top 16 bit of the version register of gp.*/ -+#define MALI200_GP_PRODUCT_ID 0xA07 -+#define MALI300_GP_PRODUCT_ID 0xC07 -+#define MALI400_GP_PRODUCT_ID 0xB07 -+#define MALI450_GP_PRODUCT_ID 0xD07 -+ -+/** -+ * The different sources for instrumented on the geometry processor. -+ * @see MALIGP2_REG_ADDR_MGMT_PERF_CNT_0_SRC -+ */ -+ -+enum MALIGP2_cont_reg_perf_cnt_src { -+ MALIGP2_REG_VAL_PERF_CNT1_SRC_NUMBER_OF_VERTICES_PROCESSED = 0x0a, -+}; -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c -new file mode 100755 -index 000000000..7df934c12 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.c -@@ -0,0 +1,13 @@ -+/* -+ * Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_timestamp.h" -+ -+/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ -diff --git a/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h -new file mode 100755 -index 000000000..f52097c19 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/timestamp-arm11-cc/mali_timestamp.h -@@ -0,0 +1,48 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_TIMESTAMP_H__ -+#define __MALI_TIMESTAMP_H__ -+ -+#include "mali_osk.h" -+ -+MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) -+{ -+ /* -+ * reset counters and overflow flags -+ */ -+ -+ u32 mask = (1 << 0) | /* enable all three counters */ -+ (0 << 1) | /* reset both Count Registers to 0x0 */ -+ (1 << 2) | /* reset the Cycle Counter Register to 0x0 */ -+ (0 << 3) | /* 1 = Cycle Counter Register counts every 64th processor clock cycle */ -+ (0 << 4) | /* Count Register 0 interrupt enable */ -+ (0 << 5) | /* Count Register 1 interrupt enable */ -+ (0 << 6) | /* Cycle Counter interrupt enable */ -+ (0 << 8) | /* Count Register 0 overflow flag (clear or write, flag on read) */ -+ (0 << 9) | /* Count Register 1 overflow flag (clear or write, flag on read) */ -+ (1 << 10); /* Cycle Counter Register overflow flag (clear or write, flag on read) */ -+ -+ __asm__ __volatile__("MCR p15, 0, %0, c15, c12, 0" : : "r"(mask)); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+MALI_STATIC_INLINE u64 _mali_timestamp_get(void) -+{ -+ u32 result; -+ -+ /* this is for the clock cycles */ -+ __asm__ __volatile__("MRC p15, 0, %0, c15, c12, 1" : "=r"(result)); -+ -+ return (u64)result; -+} -+ -+#endif /* __MALI_TIMESTAMP_H__ */ -diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c -new file mode 100755 -index 000000000..7df934c12 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.c -@@ -0,0 +1,13 @@ -+/* -+ * Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_timestamp.h" -+ -+/* This file is intentionally left empty, as all functions are inlined in mali_profiling_sampler.h */ -diff --git a/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h -new file mode 100755 -index 000000000..709a16a82 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/mali/timestamp-default/mali_timestamp.h -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __MALI_TIMESTAMP_H__ -+#define __MALI_TIMESTAMP_H__ -+ -+#include "mali_osk.h" -+ -+MALI_STATIC_INLINE _mali_osk_errcode_t _mali_timestamp_reset(void) -+{ -+ return _MALI_OSK_ERR_OK; -+} -+ -+MALI_STATIC_INLINE u64 _mali_timestamp_get(void) -+{ -+ return _mali_osk_boot_time_get_ns(); -+} -+ -+#endif /* __MALI_TIMESTAMP_H__ */ -diff --git a/drivers/gpu/arm/mali400/rk_ver_info.txt b/drivers/gpu/arm/mali400/rk_ver_info.txt -new file mode 100755 -index 000000000..2a6cbbbb5 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/rk_ver_info.txt -@@ -0,0 +1,11 @@ -+ -+r5p0-01rel0-1-x@0 -+ 对 arm_release_ver r5p0-01rel0 的定制集æˆ. -+ r5p0-01rel0 对 gpu çš„ dts 有大修改, 但这里出于兼容考虑, ä»æ—§ä½¿ç”¨ dts_for_mali_ko_befor_r5p0-01rel0. -+ -+r5p0-01rel0-2-x@0 -+ æ”¯æŒ mali_so æ¥èŽ·å– rk_ko_ver. -+ -+r5p0-01rel0-3-x@0 -+ 在 mali_control_timer_callback_chain 中使用 mod_timer, 而ä¸å†æ˜¯ add_timer. -+ -diff --git a/drivers/gpu/arm/mali400/ump/Kbuild b/drivers/gpu/arm/mali400/ump/Kbuild -new file mode 100755 -index 000000000..a3067ba72 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/Kbuild -@@ -0,0 +1,92 @@ -+# -+# Copyright (C) 2010-2012 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained from Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+# -+ -+# Set default configuration to use, if Makefile didn't provide one. -+# Change this to use a different config.h -+CONFIG ?= default -+ -+# Link arch to the selected arch-config directory -+$(shell [ -L $(src)/arch ] && rm $(src)/arch) -+$(shell ln -sf arch-$(CONFIG) $(src)/arch) -+$(shell touch $(src)/arch/config.h) -+ -+UDD_FILE_PREFIX = ../mali/ -+ -+# Get subversion revision number, fall back to 0000 if no svn info is available -+SVN_INFO = (cd $(src); svn info 2>/dev/null) -+ -+ifneq ($(shell $(SVN_INFO) 2>/dev/null),) -+# SVN detected -+SVN_REV := $(shell $(SVN_INFO) | grep '^Revision: '| sed -e 's/^Revision: //' 2>/dev/null) -+DRIVER_REV := $(MALI_RELEASE_NAME)-r$(SVN_REV) -+CHANGE_DATE := $(shell $(SVN_INFO) | grep '^Last Changed Date: ' | cut -d: -f2- | cut -b2-) -+CHANGED_REVISION := $(shell $(SVN_INFO) | grep '^Last Changed Rev: ' | cut -d: -f2- | cut -b2-) -+REPO_URL := $(shell $(SVN_INFO) | grep '^URL: ' | cut -d: -f2- | cut -b2-) -+ -+else # SVN -+GIT_REV := $(shell cd $(src); git describe --always 2>/dev/null) -+ifneq ($(GIT_REV),) -+# Git detected -+DRIVER_REV := $(MALI_RELEASE_NAME)-$(GIT_REV) -+CHANGE_DATE := $(shell cd $(src); git log -1 --format="%ci") -+CHANGED_REVISION := $(GIT_REV) -+REPO_URL := $(shell cd $(src); git describe --all --always 2>/dev/null) -+ -+else # Git -+# No Git or SVN detected -+DRIVER_REV := $(MALI_RELEASE_NAME) -+CHANGE_DATE := $(MALI_RELEASE_NAME) -+CHANGED_REVISION := $(MALI_RELEASE_NAME) -+endif -+endif -+ -+ccflags-y += -DSVN_REV=$(SVN_REV) -+ccflags-y += -DSVN_REV_STRING=\"$(DRIVER_REV)\" -+ -+ccflags-y += -I$(src) -I$(src)/common -I$(src)/linux -I$(src)/../mali/common -I$(src)/../mali/linux -I$(src)/include -I$(src)/../../ump/include/ump -+ccflags-y += -DMALI_STATE_TRACKING=0 -+ccflags-y += -DMALI_ENABLE_CPU_CYCLES=0 -+ccflags-$(CONFIG_UMP_DEBUG) += -DDEBUG -+ -+# For customer releases the Linux Device Drivers will be provided as ARM proprietary and GPL releases: -+# The ARM proprietary product will only include the license/proprietary directory -+# The GPL product will only include the license/gpl directory -+ -+ifeq ($(wildcard $(src)/linux/license/gpl/*),) -+ccflags-y += -I$(src)/linux/license/proprietary -I$(src)/../mali/linux/license/proprietary -+else -+ccflags-y += -I$(src)/linux/license/gpl -I$(src)/../mali/linux/license/gpl -+endif -+ -+ump-y = common/ump_kernel_common.o \ -+ common/ump_kernel_descriptor_mapping.o \ -+ common/ump_kernel_api.o \ -+ common/ump_kernel_ref_drv.o \ -+ linux/ump_kernel_linux.o \ -+ linux/ump_kernel_memory_backend_os.o \ -+ linux/ump_kernel_memory_backend_dedicated.o \ -+ linux/ump_memory_backend.o \ -+ linux/ump_ukk_wrappers.o \ -+ linux/ump_ukk_ref_wrappers.o \ -+ linux/ump_osk_atomics.o \ -+ linux/ump_osk_low_level_mem.o \ -+ linux/ump_osk_misc.o \ -+ linux/ump_kernel_random_mapping.o -+ -+ifneq ($(CONFIG_MALI400),y) -+ump-y += $(UDD_FILE_PREFIX)linux/mali_osk_atomics.o \ -+ $(UDD_FILE_PREFIX)linux/mali_osk_locks.o \ -+ $(UDD_FILE_PREFIX)linux/mali_osk_memory.o \ -+ $(UDD_FILE_PREFIX)linux/mali_osk_math.o \ -+ $(UDD_FILE_PREFIX)linux/mali_osk_misc.o -+endif -+ -+obj-$(CONFIG_UMP) := ump.o -+ -diff --git a/drivers/gpu/arm/mali400/ump/Kconfig b/drivers/gpu/arm/mali400/ump/Kconfig -new file mode 100755 -index 000000000..ec3509057 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/Kconfig -@@ -0,0 +1,17 @@ -+# SPDX-License-Identifier: GPL-2.0 -+config UMP -+ tristate "UMP support" -+ depends on ARM -+ help -+ This enables support for the UMP memory allocation and sharing API. -+ -+ To compile this driver as a module, choose M here: the module will be -+ called ump. -+ -+config UMP_DEBUG -+ bool "Enable extra debug in UMP" -+ depends on UMP -+ default y -+ help -+ This enabled extra debug checks and messages in UMP. -+ -diff --git a/drivers/gpu/arm/mali400/ump/Makefile b/drivers/gpu/arm/mali400/ump/Makefile -new file mode 100755 -index 000000000..88b02a22f ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/Makefile -@@ -0,0 +1,67 @@ -+# -+# Copyright (C) 2010-2012, 2014, 2016-2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained from Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+# -+ -+# For each arch check: CROSS_COMPILE , KDIR , CFLAGS += -DARCH -+ -+export ARCH ?= arm -+BUILD ?= debug -+ -+check_cc2 = \ -+ $(shell if $(1) -S -o /dev/null -xc /dev/null > /dev/null 2>&1; \ -+ then \ -+ echo "$(2)"; \ -+ else \ -+ echo "$(3)"; \ -+ fi ;) -+ -+# Check that required parameters are supplied. -+ifeq ($(CONFIG),) -+CONFIG := default -+endif -+ifeq ($(CPU)$(KDIR),) -+$(error "KDIR or CPU must be specified.") -+endif -+ -+# Get any user defined KDIR- or maybe even a hardcoded KDIR -+-include KDIR_CONFIGURATION -+ -+# Define host system directory -+KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build -+ -+ifeq ($(ARCH), arm) -+# when compiling for ARM we're cross compiling -+export CROSS_COMPILE ?= $(call check_cc2, arm-linux-gnueabi-gcc, arm-linux-gnueabi-, arm-none-linux-gnueabi-) -+endif -+ -+# look up KDIR based om CPU selection -+KDIR ?= $(KDIR-$(CPU)) -+ -+export CONFIG -+ -+export CONFIG_UMP := m -+ifeq ($(BUILD),debug) -+export CONFIG_UMP_DEBUG := y -+else -+export CONFIG_UMP_DEBUG := n -+endif -+ -+ifeq ($(KDIR),) -+$(error No KDIR found for platform $(CPU)) -+endif -+ -+all: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) modules -+ -+kernelrelease: -+ $(MAKE) -C $(KDIR) kernelrelease -+ -+clean: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) clean -+ $(MAKE) -C $(KDIR) M=$(CURDIR)/../mali clean -diff --git a/drivers/gpu/arm/mali400/ump/Makefile.common b/drivers/gpu/arm/mali400/ump/Makefile.common -new file mode 100755 -index 000000000..ad2c18da9 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/Makefile.common -@@ -0,0 +1,20 @@ -+# -+# Copyright (C) 2010-2011, 2013, 2016-2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained from Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+# -+ -+SRC = $(UMP_FILE_PREFIX)common/ump_kernel_common.c \ -+ $(UMP_FILE_PREFIX)common/ump_kernel_descriptor_mapping.c \ -+ $(UMP_FILE_PREFIX)common/ump_kernel_api.c \ -+ $(UMP_FILE_PREFIX)common/ump_kernel_ref_drv.c -+ -+# Get subversion revision number, fall back to 0000 if no svn info is available -+SVN_REV:=$(shell ((svnversion | grep -qv exported && echo -n 'Revision: ' && svnversion) || git svn info | sed -e 's/$$$$/M/' | grep '^Revision: ' || echo ${MALI_RELEASE_NAME}) 2>/dev/null | sed -e 's/^Revision: //') -+ -+EXTRA_CFLAGS += -DSVN_REV=$(SVN_REV) -+EXTRA_CFLAGS += -DSVN_REV_STRING=\"$(SVN_REV)\" -diff --git a/drivers/gpu/arm/mali400/ump/arch-default/config.h b/drivers/gpu/arm/mali400/ump/arch-default/config.h -new file mode 100755 -index 000000000..d4aef9dd0 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/arch-default/config.h -@@ -0,0 +1,24 @@ -+/* -+ * Copyright (C) 2010, 2012, 2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __ARCH_CONFIG_H__ -+#define __ARCH_CONFIG_H__ -+ -+/* Use OS memory. */ -+#define ARCH_UMP_BACKEND_DEFAULT 1 -+ -+/* OS memory won't need a base address. */ -+#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0x00000000 -+ -+/* 512 MB maximum limit for UMP allocations. */ -+#define ARCH_UMP_MEMORY_SIZE_DEFAULT 512UL * 1024UL * 1024UL -+ -+ -+#endif /* __ARCH_CONFIG_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h -new file mode 100755 -index 000000000..182e90c1d ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/arch-pb-virtex5/config.h -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __ARCH_CONFIG_H__ -+#define __ARCH_CONFIG_H__ -+ -+#define ARCH_UMP_BACKEND_DEFAULT 0 -+#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0xE1000000 -+#define ARCH_UMP_MEMORY_SIZE_DEFAULT 16UL * 1024UL * 1024UL -+ -+#endif /* __ARCH_CONFIG_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/arch/config.h b/drivers/gpu/arm/mali400/ump/arch/config.h -new file mode 100755 -index 000000000..d4aef9dd0 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/arch/config.h -@@ -0,0 +1,24 @@ -+/* -+ * Copyright (C) 2010, 2012, 2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __ARCH_CONFIG_H__ -+#define __ARCH_CONFIG_H__ -+ -+/* Use OS memory. */ -+#define ARCH_UMP_BACKEND_DEFAULT 1 -+ -+/* OS memory won't need a base address. */ -+#define ARCH_UMP_MEMORY_ADDRESS_DEFAULT 0x00000000 -+ -+/* 512 MB maximum limit for UMP allocations. */ -+#define ARCH_UMP_MEMORY_SIZE_DEFAULT 512UL * 1024UL * 1024UL -+ -+ -+#endif /* __ARCH_CONFIG_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c -new file mode 100755 -index 000000000..36adb2f53 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_api.c -@@ -0,0 +1,455 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_osk.h" -+#include "mali_osk_list.h" -+#include "ump_osk.h" -+#include "ump_uk_types.h" -+#include "ump_kernel_interface.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_random_mapping.h" -+ -+ -+ -+/* ---------------- UMP kernel space API functions follows ---------------- */ -+ -+ -+ -+UMP_KERNEL_API_EXPORT ump_secure_id ump_dd_secure_id_get(ump_dd_handle memh) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *)memh; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ DBG_MSG(5, ("Returning secure ID. ID: %u\n", mem->secure_id)); -+ -+ return mem->secure_id; -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT ump_dd_handle ump_dd_handle_create_from_secure_id(ump_secure_id secure_id) -+{ -+ ump_dd_mem *mem; -+ -+ DBG_MSG(5, ("Getting handle from secure ID. ID: %u\n", secure_id)); -+ mem = ump_random_mapping_get(device.secure_id_map, (int)secure_id); -+ if (NULL == mem) { -+ DBG_MSG(1, ("Secure ID not found. ID: %u\n", secure_id)); -+ return UMP_DD_HANDLE_INVALID; -+ } -+ -+ /* Keep the reference taken in ump_random_mapping_get() */ -+ -+ return (ump_dd_handle)mem; -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT unsigned long ump_dd_phys_block_count_get(ump_dd_handle memh) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *) memh; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ return mem->nr_blocks; -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT ump_dd_status_code ump_dd_phys_blocks_get(ump_dd_handle memh, ump_dd_physical_block *blocks, unsigned long num_blocks) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *)memh; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ if (blocks == NULL) { -+ DBG_MSG(1, ("NULL parameter in ump_dd_phys_blocks_get()\n")); -+ return UMP_DD_INVALID; -+ } -+ -+ if (mem->nr_blocks != num_blocks) { -+ DBG_MSG(1, ("Specified number of blocks do not match actual number of blocks\n")); -+ return UMP_DD_INVALID; -+ } -+ -+ DBG_MSG(5, ("Returning physical block information. ID: %u\n", mem->secure_id)); -+ -+ _mali_osk_memcpy(blocks, mem->block_array, sizeof(ump_dd_physical_block) * mem->nr_blocks); -+ -+ return UMP_DD_SUCCESS; -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT ump_dd_status_code ump_dd_phys_block_get(ump_dd_handle memh, unsigned long index, ump_dd_physical_block *block) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *)memh; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ if (block == NULL) { -+ DBG_MSG(1, ("NULL parameter in ump_dd_phys_block_get()\n")); -+ return UMP_DD_INVALID; -+ } -+ -+ if (index >= mem->nr_blocks) { -+ DBG_MSG(5, ("Invalid index specified in ump_dd_phys_block_get()\n")); -+ return UMP_DD_INVALID; -+ } -+ -+ DBG_MSG(5, ("Returning physical block information. ID: %u, index: %lu\n", mem->secure_id, index)); -+ -+ *block = mem->block_array[index]; -+ -+ return UMP_DD_SUCCESS; -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT unsigned long ump_dd_size_get(ump_dd_handle memh) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *)memh; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ DBG_MSG(5, ("Returning size. ID: %u, size: %lu\n", mem->secure_id, mem->size_bytes)); -+ -+ return mem->size_bytes; -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT void ump_dd_reference_add(ump_dd_handle memh) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *)memh; -+ int new_ref; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ new_ref = _ump_osk_atomic_inc_and_read(&mem->ref_count); -+ -+ DBG_MSG(5, ("Memory reference incremented. ID: %u, new value: %d\n", mem->secure_id, new_ref)); -+} -+ -+ -+ -+UMP_KERNEL_API_EXPORT void ump_dd_reference_release(ump_dd_handle memh) -+{ -+ ump_dd_mem *mem = (ump_dd_mem *)memh; -+ -+ DEBUG_ASSERT_POINTER(mem); -+ -+ ump_random_mapping_put(mem); -+} -+ -+ -+ -+/* --------------- Handling of user space requests follows --------------- */ -+ -+ -+_mali_osk_errcode_t _ump_uku_get_api_version(_ump_uk_api_version_s *args) -+{ -+ ump_session_data *session_data; -+ -+ DEBUG_ASSERT_POINTER(args); -+ DEBUG_ASSERT_POINTER(args->ctx); -+ -+ session_data = (ump_session_data *)args->ctx; -+ -+ /* check compatability */ -+ if (args->version == UMP_IOCTL_API_VERSION) { -+ DBG_MSG(3, ("API version set to newest %d (compatible)\n", -+ GET_VERSION(args->version))); -+ args->compatible = 1; -+ session_data->api_version = args->version; -+ } else { -+ DBG_MSG(2, ("API version set to %d (incompatible with client version %d)\n", -+ GET_VERSION(UMP_IOCTL_API_VERSION), GET_VERSION(args->version))); -+ args->compatible = 0; -+ args->version = UMP_IOCTL_API_VERSION; /* report our version */ -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+ -+_mali_osk_errcode_t _ump_ukk_release(_ump_uk_release_s *release_info) -+{ -+ ump_session_memory_list_element *session_memory_element; -+ ump_session_memory_list_element *tmp; -+ ump_session_data *session_data; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_INVALID_FUNC; -+ int secure_id; -+ -+ DEBUG_ASSERT_POINTER(release_info); -+ DEBUG_ASSERT_POINTER(release_info->ctx); -+ -+ /* Retreive the session data */ -+ session_data = (ump_session_data *)release_info->ctx; -+ -+ /* If there are many items in the memory session list we -+ * could be de-referencing this pointer a lot so keep a local copy -+ */ -+ secure_id = release_info->secure_id; -+ -+ DBG_MSG(4, ("Releasing memory with IOCTL, ID: %u\n", secure_id)); -+ -+ /* Iterate through the memory list looking for the requested secure ID */ -+ _mali_osk_mutex_wait(session_data->lock); -+ _MALI_OSK_LIST_FOREACHENTRY(session_memory_element, tmp, &session_data->list_head_session_memory_list, ump_session_memory_list_element, list) { -+ if (session_memory_element->mem->secure_id == secure_id) { -+ ump_dd_mem *release_mem; -+ -+ release_mem = session_memory_element->mem; -+ _mali_osk_list_del(&session_memory_element->list); -+ ump_dd_reference_release(release_mem); -+ _mali_osk_free(session_memory_element); -+ -+ ret = _MALI_OSK_ERR_OK; -+ break; -+ } -+ } -+ -+ _mali_osk_mutex_signal(session_data->lock); -+ DBG_MSG_IF(1, _MALI_OSK_ERR_OK != ret, ("UMP memory with ID %u does not belong to this session.\n", secure_id)); -+ -+ DBG_MSG(4, ("_ump_ukk_release() returning 0x%x\n", ret)); -+ return ret; -+} -+ -+_mali_osk_errcode_t _ump_ukk_size_get(_ump_uk_size_get_s *user_interaction) -+{ -+ ump_dd_mem *mem; -+ _mali_osk_errcode_t ret = _MALI_OSK_ERR_FAULT; -+ -+ DEBUG_ASSERT_POINTER(user_interaction); -+ -+ /* We lock the mappings so things don't get removed while we are looking for the memory */ -+ mem = ump_random_mapping_get(device.secure_id_map, user_interaction->secure_id); -+ if (NULL != mem) { -+ user_interaction->size = mem->size_bytes; -+ DBG_MSG(4, ("Returning size. ID: %u, size: %lu ", -+ (ump_secure_id)user_interaction->secure_id, -+ (unsigned long)user_interaction->size)); -+ ump_random_mapping_put(mem); -+ ret = _MALI_OSK_ERR_OK; -+ } else { -+ user_interaction->size = 0; -+ DBG_MSG(1, ("Failed to look up mapping in ump_ioctl_size_get(). ID: %u\n", -+ (ump_secure_id)user_interaction->secure_id)); -+ } -+ -+ return ret; -+} -+ -+ -+ -+void _ump_ukk_msync(_ump_uk_msync_s *args) -+{ -+ ump_dd_mem *mem = NULL; -+ void *virtual = NULL; -+ u32 size = 0; -+ u32 offset = 0; -+ -+ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); -+ if (NULL == mem) { -+ DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_msync(). ID: %u\n", -+ (ump_secure_id)args->secure_id)); -+ return; -+ } -+ -+ /* Returns the cache settings back to Userspace */ -+ args->is_cached = mem->is_cached; -+ -+ /* If this flag is the only one set, we should not do the actual flush, only the readout */ -+ if (_UMP_UK_MSYNC_READOUT_CACHE_ENABLED == args->op) { -+ DBG_MSG(3, ("_ump_ukk_msync READOUT ID: %u Enabled: %d\n", (ump_secure_id)args->secure_id, mem->is_cached)); -+ goto msync_release_and_return; -+ } -+ -+ /* Nothing to do if the memory is not caches */ -+ if (0 == mem->is_cached) { -+ DBG_MSG(3, ("_ump_ukk_msync IGNORING ID: %u Enabled: %d OP: %d\n", (ump_secure_id)args->secure_id, mem->is_cached, args->op)); -+ goto msync_release_and_return; -+ } -+ DBG_MSG(3, ("UMP[%02u] _ump_ukk_msync Flush OP: %d Address: 0x%08x Mapping: 0x%08x\n", -+ (ump_secure_id)args->secure_id, args->op, args->address, args->mapping)); -+ -+ if (args->address) { -+ virtual = (void *)((u32)args->address); -+ offset = (u32)((args->address) - (args->mapping)); -+ } else { -+ /* Flush entire mapping when no address is specified. */ -+ virtual = args->mapping; -+ } -+ if (args->size) { -+ size = args->size; -+ } else { -+ /* Flush entire mapping when no size is specified. */ -+ size = mem->size_bytes - offset; -+ } -+ -+ if ((offset + size) > mem->size_bytes) { -+ DBG_MSG(1, ("Trying to flush more than the entire UMP allocation: offset: %u + size: %u > %u\n", offset, size, mem->size_bytes)); -+ goto msync_release_and_return; -+ } -+ -+ /* The actual cache flush - Implemented for each OS*/ -+ _ump_osk_msync(mem, virtual, offset, size, args->op, NULL); -+ -+msync_release_and_return: -+ ump_random_mapping_put(mem); -+ return; -+} -+ -+void _ump_ukk_cache_operations_control(_ump_uk_cache_operations_control_s *args) -+{ -+ ump_session_data *session_data; -+ ump_uk_cache_op_control op; -+ -+ DEBUG_ASSERT_POINTER(args); -+ DEBUG_ASSERT_POINTER(args->ctx); -+ -+ op = args->op; -+ session_data = (ump_session_data *)args->ctx; -+ -+ _mali_osk_mutex_wait(session_data->lock); -+ if (op == _UMP_UK_CACHE_OP_START) { -+ session_data->cache_operations_ongoing++; -+ DBG_MSG(4, ("Cache ops start\n")); -+ if (session_data->cache_operations_ongoing != 1) { -+ DBG_MSG(2, ("UMP: Number of simultanious cache control ops: %d\n", session_data->cache_operations_ongoing)); -+ } -+ } else if (op == _UMP_UK_CACHE_OP_FINISH) { -+ DBG_MSG(4, ("Cache ops finish\n")); -+ session_data->cache_operations_ongoing--; -+#if 0 -+ if (session_data->has_pending_level1_cache_flush) { -+ /* This function will set has_pending_level1_cache_flush=0 */ -+ _ump_osk_msync(NULL, NULL, 0, 0, _UMP_UK_MSYNC_FLUSH_L1, session_data); -+ } -+#endif -+ -+ /* to be on the safe side: always flush l1 cache when cache operations are done */ -+ _ump_osk_msync(NULL, NULL, 0, 0, _UMP_UK_MSYNC_FLUSH_L1, session_data); -+ DBG_MSG(4, ("Cache ops finish end\n")); -+ } else { -+ DBG_MSG(1, ("Illegal call to %s at line %d\n", __FUNCTION__, __LINE__)); -+ } -+ _mali_osk_mutex_signal(session_data->lock); -+ -+} -+ -+void _ump_ukk_switch_hw_usage(_ump_uk_switch_hw_usage_s *args) -+{ -+ ump_dd_mem *mem = NULL; -+ ump_uk_user old_user; -+ ump_uk_msync_op cache_op = _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE; -+ ump_session_data *session_data; -+ -+ DEBUG_ASSERT_POINTER(args); -+ DEBUG_ASSERT_POINTER(args->ctx); -+ -+ session_data = (ump_session_data *)args->ctx; -+ -+ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); -+ if (NULL == mem) { -+ DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_switch_hw_usage(). ID: %u\n", -+ (ump_secure_id)args->secure_id)); -+ return; -+ } -+ -+ old_user = mem->hw_device; -+ mem->hw_device = args->new_user; -+ -+ DBG_MSG(3, ("UMP[%02u] Switch usage Start New: %s Prev: %s.\n", -+ (ump_secure_id)args->secure_id, -+ args->new_user ? "MALI" : "CPU", -+ old_user ? "MALI" : "CPU")); -+ -+ if (!mem->is_cached) { -+ DBG_MSG(3, ("UMP[%02u] Changing owner of uncached memory. Cache flushing not needed.\n", -+ (ump_secure_id)args->secure_id)); -+ goto out; -+ } -+ -+ if (old_user == args->new_user) { -+ DBG_MSG(4, ("UMP[%02u] Setting the new_user equal to previous for. Cache flushing not needed.\n", -+ (ump_secure_id)args->secure_id)); -+ goto out; -+ } -+ if ( -+ /* Previous AND new is both different from CPU */ -+ (old_user != _UMP_UK_USED_BY_CPU) && (args->new_user != _UMP_UK_USED_BY_CPU) -+ ) { -+ DBG_MSG(4, ("UMP[%02u] Previous and new user is not CPU. Cache flushing not needed.\n", -+ (ump_secure_id)args->secure_id)); -+ goto out; -+ } -+ -+ if ((old_user != _UMP_UK_USED_BY_CPU) && (args->new_user == _UMP_UK_USED_BY_CPU)) { -+ cache_op = _UMP_UK_MSYNC_INVALIDATE; -+ DBG_MSG(4, ("UMP[%02u] Cache invalidation needed\n", (ump_secure_id)args->secure_id)); -+#ifdef UMP_SKIP_INVALIDATION -+#error -+ DBG_MSG(4, ("UMP[%02u] Performing Cache invalidation SKIPPED\n", (ump_secure_id)args->secure_id)); -+ goto out; -+#endif -+ } -+ -+ /* Take lock to protect: session->cache_operations_ongoing and session->has_pending_level1_cache_flush */ -+ _mali_osk_mutex_wait(session_data->lock); -+ /* Actual cache flush */ -+ _ump_osk_msync(mem, NULL, 0, mem->size_bytes, cache_op, session_data); -+ _mali_osk_mutex_signal(session_data->lock); -+ -+out: -+ ump_random_mapping_put(mem); -+ DBG_MSG(4, ("UMP[%02u] Switch usage Finish\n", (ump_secure_id)args->secure_id)); -+ return; -+} -+ -+void _ump_ukk_lock(_ump_uk_lock_s *args) -+{ -+ ump_dd_mem *mem = NULL; -+ -+ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); -+ if (NULL == mem) { -+ DBG_MSG(1, ("UMP[%02u] Failed to look up mapping in _ump_ukk_lock(). ID: %u\n", -+ (ump_secure_id)args->secure_id)); -+ return; -+ } -+ -+ DBG_MSG(1, ("UMP[%02u] Lock. New lock flag: %d. Old Lock flag:\n", (u32)args->secure_id, (u32)args->lock_usage, (u32) mem->lock_usage)); -+ -+ mem->lock_usage = (ump_lock_usage) args->lock_usage; -+ -+ ump_random_mapping_put(mem); -+} -+ -+void _ump_ukk_unlock(_ump_uk_unlock_s *args) -+{ -+ ump_dd_mem *mem = NULL; -+ -+ mem = ump_random_mapping_get(device.secure_id_map, (int)args->secure_id); -+ if (NULL == mem) { -+ DBG_MSG(1, ("Failed to look up mapping in _ump_ukk_unlock(). ID: %u\n", -+ (ump_secure_id)args->secure_id)); -+ return; -+ } -+ -+ DBG_MSG(1, ("UMP[%02u] Unlocking. Old Lock flag:\n", -+ (u32)args->secure_id, (u32) mem->lock_usage)); -+ -+ mem->lock_usage = (ump_lock_usage) UMP_NOT_LOCKED; -+ -+ ump_random_mapping_put(mem); -+} -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c -new file mode 100755 -index 000000000..73aa9e4c4 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.c -@@ -0,0 +1,358 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_osk_bitops.h" -+#include "mali_osk_list.h" -+#include "ump_osk.h" -+#include "ump_uk_types.h" -+#include "ump_ukk.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_descriptor_mapping.h" -+#include "ump_kernel_memory_backend.h" -+ -+ -+ -+/** -+ * Define the initial and maximum size of number of secure_ids on the system -+ */ -+#define UMP_SECURE_ID_TABLE_ENTRIES_INITIAL (128 ) -+#define UMP_SECURE_ID_TABLE_ENTRIES_MAXIMUM (4096 ) -+ -+ -+/** -+ * Define the initial and maximum size of the ump_session_data::cookies_map, -+ * which is a \ref ump_descriptor_mapping. This limits how many secure_ids -+ * may be mapped into a particular process using _ump_ukk_map_mem(). -+ */ -+ -+#define UMP_COOKIES_PER_SESSION_INITIAL (UMP_SECURE_ID_TABLE_ENTRIES_INITIAL ) -+#define UMP_COOKIES_PER_SESSION_MAXIMUM (UMP_SECURE_ID_TABLE_ENTRIES_MAXIMUM) -+ -+struct ump_dev device; -+ -+_mali_osk_errcode_t ump_kernel_constructor(void) -+{ -+ _mali_osk_errcode_t err; -+ -+ /* Perform OS Specific initialization */ -+ err = _ump_osk_init(); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("Failed to initiaze the UMP Device Driver")); -+ return err; -+ } -+ -+ /* Init the global device */ -+ _mali_osk_memset(&device, 0, sizeof(device)); -+ -+ /* Create the descriptor map, which will be used for mapping secure ID to ump_dd_mem structs */ -+ device.secure_id_map = ump_random_mapping_create(); -+ if (NULL == device.secure_id_map) { -+ MSG_ERR(("Failed to create secure id lookup table\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ /* Init memory backend */ -+ device.backend = ump_memory_backend_create(); -+ if (NULL == device.backend) { -+ MSG_ERR(("Failed to create memory backend\n")); -+ ump_random_mapping_destroy(device.secure_id_map); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void ump_kernel_destructor(void) -+{ -+ DEBUG_ASSERT_POINTER(device.secure_id_map); -+ -+ ump_random_mapping_destroy(device.secure_id_map); -+ device.secure_id_map = NULL; -+ -+ device.backend->shutdown(device.backend); -+ device.backend = NULL; -+ -+ ump_memory_backend_destroy(); -+ -+ _ump_osk_term(); -+} -+ -+/** Creates a new UMP session -+ */ -+_mali_osk_errcode_t _ump_ukk_open(void **context) -+{ -+ struct ump_session_data *session_data; -+ -+ /* allocated struct to track this session */ -+ session_data = (struct ump_session_data *)_mali_osk_malloc(sizeof(struct ump_session_data)); -+ if (NULL == session_data) { -+ MSG_ERR(("Failed to allocate ump_session_data in ump_file_open()\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ session_data->lock = _mali_osk_mutex_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); -+ if (NULL == session_data->lock) { -+ MSG_ERR(("Failed to initialize lock for ump_session_data in ump_file_open()\n")); -+ _mali_osk_free(session_data); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ session_data->cookies_map = ump_descriptor_mapping_create( -+ UMP_COOKIES_PER_SESSION_INITIAL, -+ UMP_COOKIES_PER_SESSION_MAXIMUM); -+ -+ if (NULL == session_data->cookies_map) { -+ MSG_ERR(("Failed to create descriptor mapping for _ump_ukk_map_mem cookies\n")); -+ -+ _mali_osk_mutex_term(session_data->lock); -+ _mali_osk_free(session_data); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ _MALI_OSK_INIT_LIST_HEAD(&session_data->list_head_session_memory_list); -+ -+ _MALI_OSK_INIT_LIST_HEAD(&session_data->list_head_session_memory_mappings_list); -+ -+ /* Since initial version of the UMP interface did not use the API_VERSION ioctl we have to assume -+ that it is this version, and not the "latest" one: UMP_IOCTL_API_VERSION -+ Current and later API versions would do an additional call to this IOCTL and update this variable -+ to the correct one.*/ -+ session_data->api_version = MAKE_VERSION_ID(1); -+ -+ *context = (void *)session_data; -+ -+ session_data->cache_operations_ongoing = 0 ; -+ session_data->has_pending_level1_cache_flush = 0; -+ -+ DBG_MSG(2, ("New session opened\n")); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _ump_ukk_close(void **context) -+{ -+ struct ump_session_data *session_data; -+ ump_session_memory_list_element *item; -+ ump_session_memory_list_element *tmp; -+ -+ session_data = (struct ump_session_data *)*context; -+ if (NULL == session_data) { -+ MSG_ERR(("Session data is NULL in _ump_ukk_close()\n")); -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+ -+ /* Unmap any descriptors mapped in. */ -+ if (0 == _mali_osk_list_empty(&session_data->list_head_session_memory_mappings_list)) { -+ ump_memory_allocation *descriptor; -+ ump_memory_allocation *temp; -+ -+ DBG_MSG(1, ("Memory mappings found on session usage list during session termination\n")); -+ -+ /* use the 'safe' list iterator, since freeing removes the active block from the list we're iterating */ -+ _MALI_OSK_LIST_FOREACHENTRY(descriptor, temp, &session_data->list_head_session_memory_mappings_list, ump_memory_allocation, list) { -+ _ump_uk_unmap_mem_s unmap_args; -+ DBG_MSG(4, ("Freeing block with phys address 0x%x size 0x%x mapped in user space at 0x%x\n", -+ descriptor->phys_addr, descriptor->size, descriptor->mapping)); -+ unmap_args.ctx = (void *)session_data; -+ unmap_args.mapping = descriptor->mapping; -+ unmap_args.size = descriptor->size; -+ unmap_args._ukk_private = NULL; /* NOTE: unused */ -+ unmap_args.cookie = descriptor->cookie; -+ -+ /* NOTE: This modifies the list_head_session_memory_mappings_list */ -+ _ump_ukk_unmap_mem(&unmap_args); -+ } -+ } -+ -+ /* ASSERT that we really did free everything, because _ump_ukk_unmap_mem() -+ * can fail silently. */ -+ DEBUG_ASSERT(_mali_osk_list_empty(&session_data->list_head_session_memory_mappings_list)); -+ -+ _MALI_OSK_LIST_FOREACHENTRY(item, tmp, &session_data->list_head_session_memory_list, ump_session_memory_list_element, list) { -+ _mali_osk_list_del(&item->list); -+ DBG_MSG(2, ("Releasing UMP memory %u as part of file close\n", item->mem->secure_id)); -+ ump_dd_reference_release(item->mem); -+ _mali_osk_free(item); -+ } -+ -+ ump_descriptor_mapping_destroy(session_data->cookies_map); -+ -+ _mali_osk_mutex_term(session_data->lock); -+ _mali_osk_free(session_data); -+ -+ DBG_MSG(2, ("Session closed\n")); -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _ump_ukk_map_mem(_ump_uk_map_mem_s *args) -+{ -+ struct ump_session_data *session_data; -+ ump_memory_allocation *descriptor; /* Describes current mapping of memory */ -+ _mali_osk_errcode_t err; -+ unsigned long offset = 0; -+ unsigned long left; -+ ump_dd_handle handle; /* The real UMP handle for this memory. Its real datatype is ump_dd_mem* */ -+ ump_dd_mem *mem; /* The real UMP memory. It is equal to the handle, but with exposed struct */ -+ u32 block; -+ int map_id; -+ -+ session_data = (ump_session_data *)args->ctx; -+ if (NULL == session_data) { -+ MSG_ERR(("Session data is NULL in _ump_ukk_map_mem()\n")); -+ return _MALI_OSK_ERR_INVALID_ARGS; -+ } -+ -+ descriptor = (ump_memory_allocation *) _mali_osk_calloc(1, sizeof(ump_memory_allocation)); -+ if (NULL == descriptor) { -+ MSG_ERR(("ump_ukk_map_mem: descriptor allocation failed\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ handle = ump_dd_handle_create_from_secure_id(args->secure_id); -+ if (UMP_DD_HANDLE_INVALID == handle) { -+ _mali_osk_free(descriptor); -+ DBG_MSG(1, ("Trying to map unknown secure ID %u\n", args->secure_id)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ mem = (ump_dd_mem *)handle; -+ DEBUG_ASSERT(mem); -+ if (mem->size_bytes != args->size) { -+ _mali_osk_free(descriptor); -+ ump_dd_reference_release(handle); -+ DBG_MSG(1, ("Trying to map too much or little. ID: %u, virtual size=%lu, UMP size: %lu\n", args->secure_id, args->size, mem->size_bytes)); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ map_id = ump_descriptor_mapping_allocate_mapping(session_data->cookies_map, (void *) descriptor); -+ -+ if (map_id < 0) { -+ _mali_osk_free(descriptor); -+ ump_dd_reference_release(handle); -+ DBG_MSG(1, ("ump_ukk_map_mem: unable to allocate a descriptor_mapping for return cookie\n")); -+ -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ descriptor->size = args->size; -+ descriptor->handle = handle; -+ descriptor->phys_addr = args->phys_addr; -+ descriptor->process_mapping_info = args->_ukk_private; -+ descriptor->ump_session = session_data; -+ descriptor->cookie = (u32)map_id; -+ -+ if (mem->is_cached) { -+ descriptor->is_cached = 1; -+ DBG_MSG(3, ("Mapping UMP secure_id: %d as cached.\n", args->secure_id)); -+ } else { -+ descriptor->is_cached = 0; -+ DBG_MSG(3, ("Mapping UMP secure_id: %d as Uncached.\n", args->secure_id)); -+ } -+ -+ _mali_osk_list_init(&descriptor->list); -+ -+ err = _ump_osk_mem_mapregion_init(descriptor); -+ if (_MALI_OSK_ERR_OK != err) { -+ DBG_MSG(1, ("Failed to initialize memory mapping in _ump_ukk_map_mem(). ID: %u\n", args->secure_id)); -+ ump_descriptor_mapping_free(session_data->cookies_map, map_id); -+ _mali_osk_free(descriptor); -+ ump_dd_reference_release(mem); -+ return err; -+ } -+ -+ DBG_MSG(4, ("Mapping virtual to physical memory: ID: %u, size:%lu, first physical addr: 0x%08lx, number of regions: %lu\n", -+ mem->secure_id, -+ mem->size_bytes, -+ ((NULL != mem->block_array) ? mem->block_array->addr : 0), -+ mem->nr_blocks)); -+ -+ left = descriptor->size; -+ /* loop over all blocks and map them in */ -+ for (block = 0; block < mem->nr_blocks; block++) { -+ unsigned long size_to_map; -+ -+ if (left > mem->block_array[block].size) { -+ size_to_map = mem->block_array[block].size; -+ } else { -+ size_to_map = left; -+ } -+ -+ if (_MALI_OSK_ERR_OK != _ump_osk_mem_mapregion_map(descriptor, offset, (u32 *) & (mem->block_array[block].addr), size_to_map)) { -+ DBG_MSG(1, ("WARNING: _ump_ukk_map_mem failed to map memory into userspace\n")); -+ ump_descriptor_mapping_free(session_data->cookies_map, map_id); -+ ump_dd_reference_release(mem); -+ _ump_osk_mem_mapregion_term(descriptor); -+ _mali_osk_free(descriptor); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ left -= size_to_map; -+ offset += size_to_map; -+ } -+ -+ /* Add to the ump_memory_allocation tracking list */ -+ _mali_osk_mutex_wait(session_data->lock); -+ _mali_osk_list_add(&descriptor->list, &session_data->list_head_session_memory_mappings_list); -+ _mali_osk_mutex_signal(session_data->lock); -+ -+ args->mapping = descriptor->mapping; -+ args->cookie = descriptor->cookie; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _ump_ukk_unmap_mem(_ump_uk_unmap_mem_s *args) -+{ -+ struct ump_session_data *session_data; -+ ump_memory_allocation *descriptor; -+ ump_dd_handle handle; -+ -+ session_data = (ump_session_data *)args->ctx; -+ -+ if (NULL == session_data) { -+ MSG_ERR(("Session data is NULL in _ump_ukk_map_mem()\n")); -+ return; -+ } -+ -+ if (0 != ump_descriptor_mapping_get(session_data->cookies_map, (int)args->cookie, (void **)&descriptor)) { -+ MSG_ERR(("_ump_ukk_map_mem: cookie 0x%X not found for this session\n", args->cookie)); -+ return; -+ } -+ -+ DEBUG_ASSERT_POINTER(descriptor); -+ -+ handle = descriptor->handle; -+ if (UMP_DD_HANDLE_INVALID == handle) { -+ DBG_MSG(1, ("WARNING: Trying to unmap unknown handle: UNKNOWN\n")); -+ return; -+ } -+ -+ /* Remove the ump_memory_allocation from the list of tracked mappings */ -+ _mali_osk_mutex_wait(session_data->lock); -+ _mali_osk_list_del(&descriptor->list); -+ _mali_osk_mutex_signal(session_data->lock); -+ -+ ump_descriptor_mapping_free(session_data->cookies_map, (int)args->cookie); -+ -+ ump_dd_reference_release(handle); -+ -+ _ump_osk_mem_mapregion_term(descriptor); -+ _mali_osk_free(descriptor); -+} -+ -+u32 _ump_ukk_report_memory_usage(void) -+{ -+ if (device.backend->stat) -+ return device.backend->stat(device.backend); -+ else -+ return 0; -+} -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h -new file mode 100755 -index 000000000..aa65f1cb6 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_common.h -@@ -0,0 +1,125 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __UMP_KERNEL_COMMON_H__ -+#define __UMP_KERNEL_COMMON_H__ -+ -+#include "ump_kernel_types.h" -+#include "ump_kernel_interface.h" -+#include "ump_kernel_descriptor_mapping.h" -+#include "ump_kernel_random_mapping.h" -+#include "ump_kernel_memory_backend.h" -+ -+ -+#ifdef DEBUG -+extern int ump_debug_level; -+#define UMP_DEBUG_PRINT(args) _mali_osk_dbgmsg args -+#define UMP_DEBUG_CODE(args) args -+#define DBG_MSG(level,args) do { /* args should be in brackets */ \ -+ ((level) <= ump_debug_level)?\ -+ UMP_DEBUG_PRINT(("UMP<" #level ">: ")), \ -+ UMP_DEBUG_PRINT(args):0; \ -+ } while (0) -+ -+#define DBG_MSG_IF(level,condition,args) /* args should be in brackets */ \ -+ if((condition)&&((level) <= ump_debug_level)) {\ -+ UMP_DEBUG_PRINT(("UMP<" #level ">: ")); \ -+ UMP_DEBUG_PRINT(args); \ -+ } -+ -+#define DBG_MSG_ELSE(level,args) /* args should be in brackets */ \ -+ else if((level) <= ump_debug_level) { \ -+ UMP_DEBUG_PRINT(("UMP<" #level ">: ")); \ -+ UMP_DEBUG_PRINT(args); \ -+ } -+ -+#define DEBUG_ASSERT_POINTER(pointer) do {if( (pointer)== NULL) MSG_ERR(("NULL pointer " #pointer)); } while(0) -+#define DEBUG_ASSERT(condition) do {if(!(condition)) MSG_ERR(("ASSERT failed: " #condition)); } while(0) -+#else /* DEBUG */ -+#define UMP_DEBUG_PRINT(args) do {} while(0) -+#define UMP_DEBUG_CODE(args) -+#define DBG_MSG(level,args) do {} while(0) -+#define DBG_MSG_IF(level,condition,args) do {} while(0) -+#define DBG_MSG_ELSE(level,args) do {} while(0) -+#define DEBUG_ASSERT(condition) do {} while(0) -+#define DEBUG_ASSERT_POINTER(pointer) do {} while(0) -+#endif /* DEBUG */ -+ -+#define MSG_ERR(args) do{ /* args should be in brackets */ \ -+ _mali_osk_dbgmsg("UMP: ERR: %s\n" ,__FILE__); \ -+ _mali_osk_dbgmsg( " %s()%4d\n", __FUNCTION__, __LINE__) ; \ -+ _mali_osk_dbgmsg args ; \ -+ _mali_osk_dbgmsg("\n"); \ -+ } while(0) -+ -+#define MSG(args) do{ /* args should be in brackets */ \ -+ _mali_osk_dbgmsg("UMP: "); \ -+ _mali_osk_dbgmsg args; \ -+ } while (0) -+ -+ -+ -+/* -+ * This struct is used to store per session data. -+ * A session is created when someone open() the device, and -+ * closed when someone close() it or the user space application terminates. -+ */ -+typedef struct ump_session_data { -+ _mali_osk_list_t list_head_session_memory_list; /**< List of ump allocations made by the process (elements are ump_session_memory_list_element) */ -+ _mali_osk_list_t list_head_session_memory_mappings_list; /**< List of ump_memory_allocations mapped in */ -+ int api_version; -+ _mali_osk_mutex_t *lock; -+ ump_descriptor_mapping *cookies_map; /**< Secure mapping of cookies from _ump_ukk_map_mem() */ -+ int cache_operations_ongoing; -+ int has_pending_level1_cache_flush; -+} ump_session_data; -+ -+ -+ -+/* -+ * This struct is used to track the UMP memory references a session has. -+ * We need to track this in order to be able to clean up after user space processes -+ * which don't do it themself (e.g. due to a crash or premature termination). -+ */ -+typedef struct ump_session_memory_list_element { -+ struct ump_dd_mem *mem; -+ _mali_osk_list_t list; -+} ump_session_memory_list_element; -+ -+ -+ -+/* -+ * Device specific data, created when device driver is loaded, and then kept as the global variable device. -+ */ -+typedef struct ump_dev { -+ ump_random_mapping *secure_id_map; -+ ump_memory_backend *backend; -+} ump_dev; -+ -+ -+ -+extern int ump_debug_level; -+extern struct ump_dev device; -+ -+_mali_osk_errcode_t ump_kernel_constructor(void); -+void ump_kernel_destructor(void); -+int ump_map_errcode(_mali_osk_errcode_t err); -+ -+/** -+ * variables from user space cannot be dereferenced from kernel space; tagging them -+ * with __user allows the GCC compiler to generate a warning. Other compilers may -+ * not support this so we define it here as an empty macro if the compiler doesn't -+ * define it. -+ */ -+#ifndef __user -+#define __user -+#endif -+ -+#endif /* __UMP_KERNEL_COMMON_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c -new file mode 100755 -index 000000000..e4642f039 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.c -@@ -0,0 +1,155 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "mali_osk_bitops.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_descriptor_mapping.h" -+ -+#define MALI_PAD_INT(x) (((x) + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1)) -+ -+/** -+ * Allocate a descriptor table capable of holding 'count' mappings -+ * @param count Number of mappings in the table -+ * @return Pointer to a new table, NULL on error -+ */ -+static ump_descriptor_table *descriptor_table_alloc(int count); -+ -+/** -+ * Free a descriptor table -+ * @param table The table to free -+ */ -+static void descriptor_table_free(ump_descriptor_table *table); -+ -+ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries) -+{ -+ ump_descriptor_mapping *map = _mali_osk_calloc(1, sizeof(ump_descriptor_mapping)); -+ -+ init_entries = MALI_PAD_INT(init_entries); -+ max_entries = MALI_PAD_INT(max_entries); -+ -+ if (NULL != map) { -+ map->table = descriptor_table_alloc(init_entries); -+ if (NULL != map->table) { -+ map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_UNORDERED, 0); -+ if (NULL != map->lock) { -+ _mali_osk_set_nonatomic_bit(0, map->table->usage); /* reserve bit 0 to prevent NULL/zero logic to kick in */ -+ map->max_nr_mappings_allowed = max_entries; -+ map->current_nr_mappings = init_entries; -+ return map; -+ } -+ descriptor_table_free(map->table); -+ } -+ _mali_osk_free(map); -+ } -+ return NULL; -+} -+ -+void ump_descriptor_mapping_destroy(ump_descriptor_mapping *map) -+{ -+ descriptor_table_free(map->table); -+ _mali_osk_mutex_rw_term(map->lock); -+ _mali_osk_free(map); -+} -+ -+int ump_descriptor_mapping_allocate_mapping(ump_descriptor_mapping *map, void *target) -+{ -+ int descriptor = -1;/*-EFAULT;*/ -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); -+ descriptor = _mali_osk_find_first_zero_bit(map->table->usage, map->current_nr_mappings); -+ if (descriptor == map->current_nr_mappings) { -+ int nr_mappings_new; -+ /* no free descriptor, try to expand the table */ -+ ump_descriptor_table *new_table; -+ ump_descriptor_table *old_table = map->table; -+ nr_mappings_new = map->current_nr_mappings * 2; -+ -+ if (map->current_nr_mappings >= map->max_nr_mappings_allowed) { -+ descriptor = -1; -+ goto unlock_and_exit; -+ } -+ -+ new_table = descriptor_table_alloc(nr_mappings_new); -+ if (NULL == new_table) { -+ descriptor = -1; -+ goto unlock_and_exit; -+ } -+ -+ _mali_osk_memcpy(new_table->usage, old_table->usage, (sizeof(unsigned long)*map->current_nr_mappings) / BITS_PER_LONG); -+ _mali_osk_memcpy(new_table->mappings, old_table->mappings, map->current_nr_mappings * sizeof(void *)); -+ map->table = new_table; -+ map->current_nr_mappings = nr_mappings_new; -+ descriptor_table_free(old_table); -+ } -+ -+ /* we have found a valid descriptor, set the value and usage bit */ -+ _mali_osk_set_nonatomic_bit(descriptor, map->table->usage); -+ map->table->mappings[descriptor] = target; -+ -+unlock_and_exit: -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); -+ return descriptor; -+} -+ -+int ump_descriptor_mapping_get(ump_descriptor_mapping *map, int descriptor, void **target) -+{ -+ int result = -1;/*-EFAULT;*/ -+ DEBUG_ASSERT(map); -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); -+ if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { -+ *target = map->table->mappings[descriptor]; -+ result = 0; -+ } else *target = NULL; -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); -+ return result; -+} -+ -+int ump_descriptor_mapping_set(ump_descriptor_mapping *map, int descriptor, void *target) -+{ -+ int result = -1;/*-EFAULT;*/ -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); -+ if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { -+ map->table->mappings[descriptor] = target; -+ result = 0; -+ } -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); -+ return result; -+} -+ -+void ump_descriptor_mapping_free(ump_descriptor_mapping *map, int descriptor) -+{ -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); -+ if ((descriptor > 0) && (descriptor < map->current_nr_mappings) && _mali_osk_test_bit(descriptor, map->table->usage)) { -+ map->table->mappings[descriptor] = NULL; -+ _mali_osk_clear_nonatomic_bit(descriptor, map->table->usage); -+ } -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); -+} -+ -+static ump_descriptor_table *descriptor_table_alloc(int count) -+{ -+ ump_descriptor_table *table; -+ -+ table = _mali_osk_calloc(1, sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG) + (sizeof(void *) * count)); -+ -+ if (NULL != table) { -+ table->usage = (u32 *)((u8 *)table + sizeof(ump_descriptor_table)); -+ table->mappings = (void **)((u8 *)table + sizeof(ump_descriptor_table) + ((sizeof(unsigned long) * count) / BITS_PER_LONG)); -+ } -+ -+ return table; -+} -+ -+static void descriptor_table_free(ump_descriptor_table *table) -+{ -+ _mali_osk_free(table); -+} -+ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h -new file mode 100755 -index 000000000..a888ba833 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_descriptor_mapping.h -@@ -0,0 +1,89 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_kernel_descriptor_mapping.h -+ */ -+ -+#ifndef __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ -+#define __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ -+ -+#include "mali_osk.h" -+ -+/** -+ * The actual descriptor mapping table, never directly accessed by clients -+ */ -+typedef struct ump_descriptor_table { -+ u32 *usage; /**< Pointer to bitpattern indicating if a descriptor is valid/used or not */ -+ void **mappings; /**< Array of the pointers the descriptors map to */ -+} ump_descriptor_table; -+ -+/** -+ * The descriptor mapping object -+ * Provides a separate namespace where we can map an integer to a pointer -+ */ -+typedef struct ump_descriptor_mapping { -+ _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ -+ int max_nr_mappings_allowed; /**< Max number of mappings to support in this namespace */ -+ int current_nr_mappings; /**< Current number of possible mappings */ -+ ump_descriptor_table *table; /**< Pointer to the current mapping table */ -+} ump_descriptor_mapping; -+ -+/** -+ * Create a descriptor mapping object -+ * Create a descriptor mapping capable of holding init_entries growable to max_entries -+ * @param init_entries Number of entries to preallocate memory for -+ * @param max_entries Number of entries to max support -+ * @return Pointer to a descriptor mapping object, NULL on failure -+ */ -+ump_descriptor_mapping *ump_descriptor_mapping_create(int init_entries, int max_entries); -+ -+/** -+ * Destroy a descriptor mapping object -+ * @param map The map to free -+ */ -+void ump_descriptor_mapping_destroy(ump_descriptor_mapping *map); -+ -+/** -+ * Allocate a new mapping entry (descriptor ID) -+ * Allocates a new entry in the map. -+ * @param map The map to allocate a new entry in -+ * @param target The value to map to -+ * @return The descriptor allocated, a negative value on error -+ */ -+int ump_descriptor_mapping_allocate_mapping(ump_descriptor_mapping *map, void *target); -+ -+/** -+ * Get the value mapped to by a descriptor ID -+ * @param map The map to lookup the descriptor id in -+ * @param descriptor The descriptor ID to lookup -+ * @param target Pointer to a pointer which will receive the stored value -+ * @return 0 on successful lookup, negative on error -+ */ -+int ump_descriptor_mapping_get(ump_descriptor_mapping *map, int descriptor, void **target); -+ -+/** -+ * Set the value mapped to by a descriptor ID -+ * @param map The map to lookup the descriptor id in -+ * @param descriptor The descriptor ID to lookup -+ * @param target Pointer to replace the current value with -+ * @return 0 on successful lookup, negative on error -+ */ -+int ump_descriptor_mapping_set(ump_descriptor_mapping *map, int descriptor, void *target); -+ -+/** -+ * Free the descriptor ID -+ * For the descriptor to be reused it has to be freed -+ * @param map The map to free the descriptor from -+ * @param descriptor The descriptor ID to free -+ */ -+void ump_descriptor_mapping_free(ump_descriptor_mapping *map, int descriptor); -+ -+#endif /* __UMP_KERNEL_DESCRIPTOR_MAPPING_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h -new file mode 100755 -index 000000000..2b69f68e8 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_memory_backend.h -@@ -0,0 +1,48 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_kernel_memory_mapping.h -+ */ -+ -+#ifndef __UMP_KERNEL_MEMORY_BACKEND_H__ -+#define __UMP_KERNEL_MEMORY_BACKEND_H__ -+ -+#include "ump_kernel_interface.h" -+#include "ump_kernel_types.h" -+ -+ -+typedef struct ump_memory_allocation { -+ void *phys_addr; -+ void *mapping; -+ unsigned long size; -+ ump_dd_handle handle; -+ void *process_mapping_info; -+ u32 cookie; /**< necessary on some U/K interface implementations */ -+ struct ump_session_data *ump_session; /**< Session that this allocation belongs to */ -+ _mali_osk_list_t list; /**< List for linking together memory allocations into the session's memory head */ -+ u32 is_cached; -+} ump_memory_allocation; -+ -+typedef struct ump_memory_backend { -+ int (*allocate)(void *ctx, ump_dd_mem *descriptor); -+ void (*release)(void *ctx, ump_dd_mem *descriptor); -+ void (*shutdown)(struct ump_memory_backend *backend); -+ u32(*stat)(struct ump_memory_backend *backend); -+ int (*pre_allocate_physical_check)(void *ctx, u32 size); -+ u32(*adjust_to_mali_phys)(void *ctx, u32 cpu_phys); -+ void *ctx; -+} ump_memory_backend; -+ -+ump_memory_backend *ump_memory_backend_create(void); -+void ump_memory_backend_destroy(void); -+ -+#endif /*__UMP_KERNEL_MEMORY_BACKEND_H__ */ -+ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c -new file mode 100755 -index 000000000..0b6434bee ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_ref_drv.c -@@ -0,0 +1,181 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_osk.h" -+#include "mali_osk_list.h" -+#include "ump_osk.h" -+#include "ump_uk_types.h" -+ -+#include "ump_kernel_interface_ref_drv.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_descriptor_mapping.h" -+ -+#define UMP_MINIMUM_SIZE 4096 -+#define UMP_MINIMUM_SIZE_MASK (~(UMP_MINIMUM_SIZE-1)) -+#define UMP_SIZE_ALIGN(x) (((x)+UMP_MINIMUM_SIZE-1)&UMP_MINIMUM_SIZE_MASK) -+#define UMP_ADDR_ALIGN_OFFSET(x) ((x)&(UMP_MINIMUM_SIZE-1)) -+static void phys_blocks_release(void *ctx, struct ump_dd_mem *descriptor); -+ -+UMP_KERNEL_API_EXPORT ump_dd_handle ump_dd_handle_create_from_phys_blocks(ump_dd_physical_block *blocks, unsigned long num_blocks) -+{ -+ ump_dd_mem *mem; -+ unsigned long size_total = 0; -+ int ret; -+ u32 i; -+ -+ /* Go through the input blocks and verify that they are sane */ -+ for (i = 0; i < num_blocks; i++) { -+ unsigned long addr = blocks[i].addr; -+ unsigned long size = blocks[i].size; -+ -+ DBG_MSG(5, ("Adding physical memory to new handle. Address: 0x%08lx, size: %lu\n", addr, size)); -+ size_total += blocks[i].size; -+ -+ if (0 != UMP_ADDR_ALIGN_OFFSET(addr)) { -+ MSG_ERR(("Trying to create UMP memory from unaligned physical address. Address: 0x%08lx\n", addr)); -+ return UMP_DD_HANDLE_INVALID; -+ } -+ -+ if (0 != UMP_ADDR_ALIGN_OFFSET(size)) { -+ MSG_ERR(("Trying to create UMP memory with unaligned size. Size: %lu\n", size)); -+ return UMP_DD_HANDLE_INVALID; -+ } -+ } -+ -+ /* Allocate the ump_dd_mem struct for this allocation */ -+ mem = _mali_osk_malloc(sizeof(*mem)); -+ if (NULL == mem) { -+ DBG_MSG(1, ("Could not allocate ump_dd_mem in ump_dd_handle_create_from_phys_blocks()\n")); -+ return UMP_DD_HANDLE_INVALID; -+ } -+ -+ /* Now, make a copy of the block information supplied by the user */ -+ mem->block_array = _mali_osk_malloc(sizeof(ump_dd_physical_block) * num_blocks); -+ if (NULL == mem->block_array) { -+ _mali_osk_free(mem); -+ DBG_MSG(1, ("Could not allocate a mem handle for function ump_dd_handle_create_from_phys_blocks().\n")); -+ return UMP_DD_HANDLE_INVALID; -+ } -+ -+ _mali_osk_memcpy(mem->block_array, blocks, sizeof(ump_dd_physical_block) * num_blocks); -+ -+ /* And setup the rest of the ump_dd_mem struct */ -+ _mali_osk_atomic_init(&mem->ref_count, 1); -+ mem->size_bytes = size_total; -+ mem->nr_blocks = num_blocks; -+ mem->backend_info = NULL; -+ mem->ctx = NULL; -+ mem->release_func = phys_blocks_release; -+ /* For now UMP handles created by ump_dd_handle_create_from_phys_blocks() is forced to be Uncached */ -+ mem->is_cached = 0; -+ mem->hw_device = _UMP_UK_USED_BY_CPU; -+ mem->lock_usage = UMP_NOT_LOCKED; -+ -+ /* Find a secure ID for this allocation */ -+ ret = ump_random_mapping_insert(device.secure_id_map, mem); -+ if (unlikely(ret)) { -+ _mali_osk_free(mem->block_array); -+ _mali_osk_free(mem); -+ DBG_MSG(1, ("Failed to allocate secure ID in ump_dd_handle_create_from_phys_blocks()\n")); -+ return UMP_DD_HANDLE_INVALID; -+ } -+ -+ DBG_MSG(3, ("UMP memory created. ID: %u, size: %lu\n", mem->secure_id, mem->size_bytes)); -+ -+ return (ump_dd_handle)mem; -+} -+ -+static void phys_blocks_release(void *ctx, struct ump_dd_mem *descriptor) -+{ -+ _mali_osk_free(descriptor->block_array); -+ descriptor->block_array = NULL; -+} -+ -+_mali_osk_errcode_t _ump_ukk_allocate(_ump_uk_allocate_s *user_interaction) -+{ -+ ump_session_data *session_data = NULL; -+ ump_dd_mem *new_allocation = NULL; -+ ump_session_memory_list_element *session_memory_element = NULL; -+ int ret; -+ -+ DEBUG_ASSERT_POINTER(user_interaction); -+ DEBUG_ASSERT_POINTER(user_interaction->ctx); -+ -+ session_data = (ump_session_data *) user_interaction->ctx; -+ -+ session_memory_element = _mali_osk_calloc(1, sizeof(ump_session_memory_list_element)); -+ if (NULL == session_memory_element) { -+ DBG_MSG(1, ("Failed to allocate ump_session_memory_list_element in ump_ioctl_allocate()\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ -+ new_allocation = _mali_osk_calloc(1, sizeof(ump_dd_mem)); -+ if (NULL == new_allocation) { -+ _mali_osk_free(session_memory_element); -+ DBG_MSG(1, ("Failed to allocate ump_dd_mem in _ump_ukk_allocate()\n")); -+ return _MALI_OSK_ERR_NOMEM; -+ } -+ -+ /* Initialize the part of the new_allocation that we know so for */ -+ _mali_osk_atomic_init(&new_allocation->ref_count, 1); -+ if (0 == (UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE & user_interaction->constraints)) -+ new_allocation->is_cached = 0; -+ else new_allocation->is_cached = 1; -+ -+ /* Special case a size of 0, we should try to emulate what malloc does -+ * in this case, which is to return a valid pointer that must be freed, -+ * but can't be dereferenced */ -+ if (0 == user_interaction->size) { -+ /* Emulate by actually allocating the minimum block size */ -+ user_interaction->size = 1; -+ } -+ -+ /* Page align the size */ -+ new_allocation->size_bytes = UMP_SIZE_ALIGN(user_interaction->size); -+ new_allocation->lock_usage = UMP_NOT_LOCKED; -+ -+ /* Now, ask the active memory backend to do the actual memory allocation */ -+ if (!device.backend->allocate(device.backend->ctx, new_allocation)) { -+ DBG_MSG(3, ("OOM: No more UMP memory left. Failed to allocate memory in ump_ioctl_allocate(). Size: %lu, requested size: %lu\n", -+ new_allocation->size_bytes, -+ (unsigned long)user_interaction->size)); -+ _mali_osk_free(new_allocation); -+ _mali_osk_free(session_memory_element); -+ return _MALI_OSK_ERR_INVALID_FUNC; -+ } -+ new_allocation->hw_device = _UMP_UK_USED_BY_CPU; -+ new_allocation->ctx = device.backend->ctx; -+ new_allocation->release_func = device.backend->release; -+ -+ /* Initialize the session_memory_element, and add it to the session object */ -+ session_memory_element->mem = new_allocation; -+ _mali_osk_mutex_wait(session_data->lock); -+ _mali_osk_list_add(&(session_memory_element->list), &(session_data->list_head_session_memory_list)); -+ _mali_osk_mutex_signal(session_data->lock); -+ -+ /* Create a secure ID for this allocation */ -+ ret = ump_random_mapping_insert(device.secure_id_map, new_allocation); -+ if (unlikely(ret)) { -+ new_allocation->release_func(new_allocation->ctx, new_allocation); -+ _mali_osk_free(session_memory_element); -+ _mali_osk_free(new_allocation); -+ DBG_MSG(1, ("Failed to allocate secure ID in ump_ioctl_allocate()\n")); -+ return _MALI_OSK_ERR_INVALID_FUNC; -+ } -+ -+ user_interaction->secure_id = new_allocation->secure_id; -+ user_interaction->size = new_allocation->size_bytes; -+ DBG_MSG(3, ("UMP memory allocated. ID: %u, size: %lu\n", -+ new_allocation->secure_id, -+ new_allocation->size_bytes)); -+ -+ return _MALI_OSK_ERR_OK; -+} -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h -new file mode 100755 -index 000000000..32f32ccbe ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_kernel_types.h -@@ -0,0 +1,58 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __UMP_KERNEL_TYPES_H__ -+#define __UMP_KERNEL_TYPES_H__ -+ -+#include "ump_kernel_interface.h" -+#include "mali_osk.h" -+ -+#include -+#ifdef CONFIG_DMA_SHARED_BUFFER -+#include -+#endif -+ -+typedef enum { -+ UMP_USED_BY_CPU = 0, -+ UMP_USED_BY_MALI = 1, -+ UMP_USED_BY_UNKNOWN_DEVICE = 100, -+} ump_hw_usage; -+ -+typedef enum { -+ UMP_NOT_LOCKED = 0, -+ UMP_READ = 1, -+ UMP_READ_WRITE = 3, -+} ump_lock_usage; -+ -+/* -+ * This struct is what is "behind" a ump_dd_handle -+ */ -+typedef struct ump_dd_mem { -+ struct rb_node node; -+ ump_secure_id secure_id; -+ _mali_osk_atomic_t ref_count; -+ unsigned long size_bytes; -+ unsigned long nr_blocks; -+ ump_dd_physical_block *block_array; -+ void (*release_func)(void *ctx, struct ump_dd_mem *descriptor); -+ void *ctx; -+ void *backend_info; -+ int is_cached; -+ ump_hw_usage hw_device; -+ ump_lock_usage lock_usage; -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ struct dma_buf_attachment *import_attach; -+ struct sg_table *sgt; -+#endif -+} ump_dd_mem; -+ -+ -+ -+#endif /* __UMP_KERNEL_TYPES_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_osk.h b/drivers/gpu/arm/mali400/ump/common/ump_osk.h -new file mode 100755 -index 000000000..9adc4d3df ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_osk.h -@@ -0,0 +1,48 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_osk.h -+ * Defines the OS abstraction layer for the UMP kernel device driver (OSK) -+ */ -+ -+#ifndef __UMP_OSK_H__ -+#define __UMP_OSK_H__ -+ -+#include -+#include -+#include "ump_uk_types.h" -+#include "ump_kernel_common.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+_mali_osk_errcode_t _ump_osk_init(void); -+ -+_mali_osk_errcode_t _ump_osk_term(void); -+ -+int _ump_osk_atomic_inc_and_read(_mali_osk_atomic_t *atom); -+ -+int _ump_osk_atomic_dec_and_read(_mali_osk_atomic_t *atom); -+ -+_mali_osk_errcode_t _ump_osk_mem_mapregion_init(ump_memory_allocation *descriptor); -+ -+_mali_osk_errcode_t _ump_osk_mem_mapregion_map(ump_memory_allocation *descriptor, u32 offset, u32 *phys_addr, unsigned long size); -+ -+void _ump_osk_mem_mapregion_term(ump_memory_allocation *descriptor); -+ -+void _ump_osk_msync(ump_dd_mem *mem, void *virt, u32 offset, u32 size, ump_uk_msync_op op, ump_session_data *session_data); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h -new file mode 100755 -index 000000000..db842cdcb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_uk_types.h -@@ -0,0 +1,202 @@ -+/* -+ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_uk_types.h -+ * Defines the types and constants used in the user-kernel interface -+ */ -+ -+#ifndef __UMP_UK_TYPES_H__ -+#define __UMP_UK_TYPES_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/* Helpers for API version handling */ -+#define MAKE_VERSION_ID(x) (((x) << 16UL) | (x)) -+#define IS_VERSION_ID(x) (((x) & 0xFFFF) == (((x) >> 16UL) & 0xFFFF)) -+#define GET_VERSION(x) (((x) >> 16UL) & 0xFFFF) -+#define IS_API_MATCH(x, y) (IS_VERSION_ID((x)) && IS_VERSION_ID((y)) && (GET_VERSION((x)) == GET_VERSION((y)))) -+ -+/** -+ * API version define. -+ * Indicates the version of the kernel API -+ * The version is a 16bit integer incremented on each API change. -+ * The 16bit integer is stored twice in a 32bit integer -+ * So for version 1 the value would be 0x00010001 -+ */ -+#define UMP_IOCTL_API_VERSION MAKE_VERSION_ID(3) -+ -+typedef enum -+{ -+ _UMP_IOC_QUERY_API_VERSION = 1, -+ _UMP_IOC_ALLOCATE, -+ _UMP_IOC_RELEASE, -+ _UMP_IOC_SIZE_GET, -+ _UMP_IOC_MAP_MEM, /* not used in Linux */ -+ _UMP_IOC_UNMAP_MEM, /* not used in Linux */ -+ _UMP_IOC_MSYNC, -+ _UMP_IOC_CACHE_OPERATIONS_CONTROL, -+ _UMP_IOC_SWITCH_HW_USAGE, -+ _UMP_IOC_LOCK, -+ _UMP_IOC_UNLOCK, -+ _UMP_IOC_DMABUF_IMPORT, -+} _ump_uk_functions; -+ -+typedef enum -+{ -+ UMP_REF_DRV_UK_CONSTRAINT_NONE = 0, -+ UMP_REF_DRV_UK_CONSTRAINT_PHYSICALLY_LINEAR = 1, -+ UMP_REF_DRV_UK_CONSTRAINT_USE_CACHE = 4, -+} ump_uk_alloc_constraints; -+ -+typedef enum -+{ -+ _UMP_UK_MSYNC_CLEAN = 0, -+ _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE = 1, -+ _UMP_UK_MSYNC_INVALIDATE = 2, -+ _UMP_UK_MSYNC_FLUSH_L1 = 3, -+ _UMP_UK_MSYNC_READOUT_CACHE_ENABLED = 128, -+} ump_uk_msync_op; -+ -+typedef enum -+{ -+ _UMP_UK_CACHE_OP_START = 0, -+ _UMP_UK_CACHE_OP_FINISH = 1, -+} ump_uk_cache_op_control; -+ -+typedef enum -+{ -+ _UMP_UK_READ = 1, -+ _UMP_UK_READ_WRITE = 3, -+} ump_uk_lock_usage; -+ -+typedef enum -+{ -+ _UMP_UK_USED_BY_CPU = 0, -+ _UMP_UK_USED_BY_MALI = 1, -+ _UMP_UK_USED_BY_UNKNOWN_DEVICE = 100, -+} ump_uk_user; -+ -+/** -+ * Get API version ([in,out] u32 api_version, [out] u32 compatible) -+ */ -+typedef struct _ump_uk_api_version_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 version; /**< Set to the user space version on entry, stores the device driver version on exit */ -+ u32 compatible; /**< Non-null if the device is compatible with the client */ -+} _ump_uk_api_version_s; -+ -+/** -+ * ALLOCATE ([out] u32 secure_id, [in,out] u32 size, [in] contraints) -+ */ -+typedef struct _ump_uk_allocate_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 secure_id; /**< Return value from DD to Userdriver */ -+ u32 size; /**< Input and output. Requested size; input. Returned size; output */ -+ ump_uk_alloc_constraints constraints; /**< Only input to Devicedriver */ -+} _ump_uk_allocate_s; -+ -+/** -+ * SIZE_GET ([in] u32 secure_id, [out]size ) -+ */ -+typedef struct _ump_uk_size_get_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 secure_id; /**< Input to DD */ -+ u32 size; /**< Returned size; output */ -+} _ump_uk_size_get_s; -+ -+/** -+ * Release ([in] u32 secure_id) -+ */ -+typedef struct _ump_uk_release_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 secure_id; /**< Input to DD */ -+} _ump_uk_release_s; -+ -+typedef struct _ump_uk_map_mem_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ void *mapping; /**< [out] Returns user-space virtual address for the mapping */ -+ void *phys_addr; /**< [in] physical address */ -+ unsigned long size; /**< [in] size */ -+ u32 secure_id; /**< [in] secure_id to assign to mapping */ -+ void *_ukk_private; /**< Only used inside linux port between kernel frontend and common part to store vma */ -+ u32 cookie; -+ u32 is_cached; /**< [in,out] caching of CPU mappings */ -+} _ump_uk_map_mem_s; -+ -+typedef struct _ump_uk_unmap_mem_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ void *mapping; -+ u32 size; -+ void *_ukk_private; -+ u32 cookie; -+} _ump_uk_unmap_mem_s; -+ -+typedef struct _ump_uk_msync_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ void *mapping; /**< [in] mapping addr */ -+ void *address; /**< [in] flush start addr */ -+ u32 size; /**< [in] size to flush */ -+ ump_uk_msync_op op; /**< [in] flush operation */ -+ u32 cookie; /**< [in] cookie stored with reference to the kernel mapping internals */ -+ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ -+ u32 is_cached; /**< [out] caching of CPU mappings */ -+} _ump_uk_msync_s; -+ -+typedef struct _ump_uk_cache_operations_control_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ ump_uk_cache_op_control op; /**< [in] cache operations start/stop */ -+} _ump_uk_cache_operations_control_s; -+ -+ -+typedef struct _ump_uk_switch_hw_usage_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ -+ ump_uk_user new_user; /**< [in] cookie stored with reference to the kernel mapping internals */ -+ -+} _ump_uk_switch_hw_usage_s; -+ -+typedef struct _ump_uk_lock_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ -+ ump_uk_lock_usage lock_usage; -+} _ump_uk_lock_s; -+ -+typedef struct _ump_uk_unlock_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ u32 secure_id; /**< [in] secure_id that identifies the ump buffer */ -+} _ump_uk_unlock_s; -+ -+typedef struct _ump_uk_dmabuf_s -+{ -+ void *ctx; /**< [in,out] user-kernel context (trashed on output) */ -+ int fd; /**< [in] dmabuf_fd that identifies the dmabuf buffer */ -+ size_t size; /**< [in] size of the buffer */ -+ u32 secure_id; /**< [out] secure_id that identifies the ump buffer */ -+} _ump_uk_dmabuf_s; -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __UMP_UK_TYPES_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/common/ump_ukk.h b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h -new file mode 100755 -index 000000000..f2906768c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/common/ump_ukk.h -@@ -0,0 +1,60 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_ukk.h -+ * Defines the kernel-side interface of the user-kernel interface -+ */ -+ -+#ifndef __UMP_UKK_H__ -+#define __UMP_UKK_H__ -+ -+#include "mali_osk.h" -+#include "ump_uk_types.h" -+ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+ -+_mali_osk_errcode_t _ump_ukk_open(void **context); -+ -+_mali_osk_errcode_t _ump_ukk_close(void **context); -+ -+_mali_osk_errcode_t _ump_ukk_allocate(_ump_uk_allocate_s *user_interaction); -+ -+_mali_osk_errcode_t _ump_ukk_release(_ump_uk_release_s *release_info); -+ -+_mali_osk_errcode_t _ump_ukk_size_get(_ump_uk_size_get_s *user_interaction); -+ -+_mali_osk_errcode_t _ump_ukk_map_mem(_ump_uk_map_mem_s *args); -+ -+_mali_osk_errcode_t _ump_uku_get_api_version(_ump_uk_api_version_s *args); -+ -+void _ump_ukk_unmap_mem(_ump_uk_unmap_mem_s *args); -+ -+void _ump_ukk_msync(_ump_uk_msync_s *args); -+ -+void _ump_ukk_cache_operations_control(_ump_uk_cache_operations_control_s *args); -+ -+void _ump_ukk_switch_hw_usage(_ump_uk_switch_hw_usage_s *args); -+ -+void _ump_ukk_lock(_ump_uk_lock_s *args); -+ -+void _ump_ukk_unlock(_ump_uk_unlock_s *args); -+ -+u32 _ump_ukk_report_memory_usage(void); -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __UMP_UKK_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h -new file mode 100755 -index 000000000..d0174055a ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/license/gpl/ump_kernel_license.h -@@ -0,0 +1,30 @@ -+/* -+ * Copyright (C) 2010, 2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_kernel_license.h -+ * Defines for the macro MODULE_LICENSE. -+ */ -+ -+#ifndef __UMP_KERNEL_LICENSE_H__ -+#define __UMP_KERNEL_LICENSE_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#define UMP_KERNEL_LINUX_LICENSE "GPL" -+#define UMP_LICENSE_IS_GPL 1 -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __UMP_KERNEL_LICENSE_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h -new file mode 100755 -index 000000000..bfb4e8d64 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_ioctl.h -@@ -0,0 +1,54 @@ -+/* -+ * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __UMP_IOCTL_H__ -+#define __UMP_IOCTL_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include -+#include -+ -+#include -+ -+#ifndef __user -+#define __user -+#endif -+ -+ -+/** -+ * @file UMP_ioctl.h -+ * This file describes the interface needed to use the Linux device driver. -+ * The interface is used by the userpace UMP driver. -+ */ -+ -+#define UMP_IOCTL_NR 0x90 -+ -+ -+#define UMP_IOC_QUERY_API_VERSION _IOR(UMP_IOCTL_NR, _UMP_IOC_QUERY_API_VERSION, _ump_uk_api_version_s) -+#define UMP_IOC_ALLOCATE _IOWR(UMP_IOCTL_NR, _UMP_IOC_ALLOCATE, _ump_uk_allocate_s) -+#define UMP_IOC_RELEASE _IOR(UMP_IOCTL_NR, _UMP_IOC_RELEASE, _ump_uk_release_s) -+#define UMP_IOC_SIZE_GET _IOWR(UMP_IOCTL_NR, _UMP_IOC_SIZE_GET, _ump_uk_size_get_s) -+#define UMP_IOC_MSYNC _IOW(UMP_IOCTL_NR, _UMP_IOC_MSYNC, _ump_uk_msync_s) -+ -+#define UMP_IOC_CACHE_OPERATIONS_CONTROL _IOW(UMP_IOCTL_NR, _UMP_IOC_CACHE_OPERATIONS_CONTROL, _ump_uk_cache_operations_control_s) -+#define UMP_IOC_SWITCH_HW_USAGE _IOW(UMP_IOCTL_NR, _UMP_IOC_SWITCH_HW_USAGE, _ump_uk_switch_hw_usage_s) -+#define UMP_IOC_LOCK _IOW(UMP_IOCTL_NR, _UMP_IOC_LOCK, _ump_uk_lock_s) -+#define UMP_IOC_UNLOCK _IOW(UMP_IOCTL_NR, _UMP_IOC_UNLOCK, _ump_uk_unlock_s) -+ -+#define UMP_IOC_DMABUF_IMPORT _IOW(UMP_IOCTL_NR, _UMP_IOC_DMABUF_IMPORT, _ump_uk_dmabuf_s) -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __UMP_IOCTL_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c -new file mode 100755 -index 000000000..71b30830c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.c -@@ -0,0 +1,449 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include /* kernel module definitions */ -+#include /* file system operations */ -+#include /* character device definitions */ -+#include /* request_mem_region */ -+#include /* memory management functions and types */ -+#include /* user space access */ -+#include -+#include -+#include -+ -+#include "arch/config.h" /* Configuration for current platform. The symlinc for arch is set by Makefile */ -+#include "ump_ioctl.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_interface.h" -+#include "ump_kernel_interface_ref_drv.h" -+#include "ump_kernel_descriptor_mapping.h" -+#include "ump_kernel_memory_backend.h" -+#include "ump_kernel_memory_backend_os.h" -+#include "ump_kernel_memory_backend_dedicated.h" -+#include "ump_kernel_license.h" -+ -+#include "ump_osk.h" -+#include "ump_ukk.h" -+#include "ump_uk_types.h" -+#include "ump_ukk_wrappers.h" -+#include "ump_ukk_ref_wrappers.h" -+ -+ -+/* Module parameter to control log level */ -+int ump_debug_level = 2; -+module_param(ump_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ -+MODULE_PARM_DESC(ump_debug_level, "Higher number, more dmesg output"); -+ -+/* By default the module uses any available major, but it's possible to set it at load time to a specific number */ -+int ump_major = 0; -+module_param(ump_major, int, S_IRUGO); /* r--r--r-- */ -+MODULE_PARM_DESC(ump_major, "Device major number"); -+ -+/* Name of the UMP device driver */ -+static char ump_dev_name[] = "ump"; /* should be const, but the functions we call requires non-cost */ -+ -+ -+#if UMP_LICENSE_IS_GPL -+static struct dentry *ump_debugfs_dir = NULL; -+#endif -+ -+/* -+ * The data which we attached to each virtual memory mapping request we get. -+ * Each memory mapping has a reference to the UMP memory it maps. -+ * We release this reference when the last memory mapping is unmapped. -+ */ -+typedef struct ump_vma_usage_tracker { -+ int references; -+ ump_dd_handle handle; -+} ump_vma_usage_tracker; -+ -+struct ump_device { -+ struct cdev cdev; -+#if UMP_LICENSE_IS_GPL -+ struct class *ump_class; -+#endif -+}; -+ -+/* The global variable containing the global device data */ -+static struct ump_device ump_device; -+struct device *ump_global_mdev = NULL; -+ -+/* Forward declare static functions */ -+static int ump_file_open(struct inode *inode, struct file *filp); -+static int ump_file_release(struct inode *inode, struct file *filp); -+#ifdef HAVE_UNLOCKED_IOCTL -+static long ump_file_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); -+#else -+static int ump_file_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg); -+#endif -+static int ump_file_mmap(struct file *filp, struct vm_area_struct *vma); -+ -+ -+/* This variable defines the file operations this UMP device driver offer */ -+static struct file_operations ump_fops = { -+ .owner = THIS_MODULE, -+ .open = ump_file_open, -+ .release = ump_file_release, -+#ifdef HAVE_UNLOCKED_IOCTL -+ .unlocked_ioctl = ump_file_ioctl, -+#else -+ .ioctl = ump_file_ioctl, -+#endif -+ .mmap = ump_file_mmap -+}; -+ -+ -+/* This function is called by Linux to initialize this module. -+ * All we do is initialize the UMP device driver. -+ */ -+static int ump_initialize_module(void) -+{ -+ _mali_osk_errcode_t err; -+ -+ DBG_MSG(2, ("Inserting UMP device driver. Compiled: %s, time: %s\n", __DATE__, __TIME__)); -+ -+ err = ump_kernel_constructor(); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("UMP device driver init failed\n")); -+ return ump_map_errcode(err); -+ } -+ -+ MSG(("UMP device driver %s loaded\n", SVN_REV_STRING)); -+ return 0; -+} -+ -+ -+ -+/* -+ * This function is called by Linux to unload/terminate/exit/cleanup this module. -+ * All we do is terminate the UMP device driver. -+ */ -+static void ump_cleanup_module(void) -+{ -+ DBG_MSG(2, ("Unloading UMP device driver\n")); -+ ump_kernel_destructor(); -+ DBG_MSG(2, ("Module unloaded\n")); -+} -+ -+ -+ -+static ssize_t ump_memory_used_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ size_t r; -+ u32 mem = _ump_ukk_report_memory_usage(); -+ -+ r = snprintf(buf, 64, "%u\n", mem); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static const struct file_operations ump_memory_usage_fops = { -+ .owner = THIS_MODULE, -+ .read = ump_memory_used_read, -+}; -+ -+/* -+ * Initialize the UMP device driver. -+ */ -+int ump_kernel_device_initialize(void) -+{ -+ int err; -+ dev_t dev = 0; -+#if UMP_LICENSE_IS_GPL -+ ump_debugfs_dir = debugfs_create_dir(ump_dev_name, NULL); -+ if (ERR_PTR(-ENODEV) == ump_debugfs_dir) { -+ ump_debugfs_dir = NULL; -+ } else { -+ debugfs_create_file("memory_usage", 0400, ump_debugfs_dir, NULL, &ump_memory_usage_fops); -+ } -+#endif -+ -+ if (0 == ump_major) { -+ /* auto select a major */ -+ err = alloc_chrdev_region(&dev, 0, 1, ump_dev_name); -+ ump_major = MAJOR(dev); -+ } else { -+ /* use load time defined major number */ -+ dev = MKDEV(ump_major, 0); -+ err = register_chrdev_region(dev, 1, ump_dev_name); -+ } -+ -+ if (0 == err) { -+ memset(&ump_device, 0, sizeof(ump_device)); -+ -+ /* initialize our char dev data */ -+ cdev_init(&ump_device.cdev, &ump_fops); -+ ump_device.cdev.owner = THIS_MODULE; -+ ump_device.cdev.ops = &ump_fops; -+ -+ /* register char dev with the kernel */ -+ err = cdev_add(&ump_device.cdev, dev, 1/*count*/); -+ if (0 == err) { -+ -+#if UMP_LICENSE_IS_GPL -+ ump_device.ump_class = class_create(THIS_MODULE, ump_dev_name); -+ if (IS_ERR(ump_device.ump_class)) { -+ err = PTR_ERR(ump_device.ump_class); -+ } else { -+ ump_global_mdev = device_create(ump_device.ump_class, NULL, dev, NULL, ump_dev_name); -+ if (!IS_ERR(ump_global_mdev)) { -+ return 0; -+ } -+ -+ err = PTR_ERR(ump_global_mdev); -+ } -+ cdev_del(&ump_device.cdev); -+#else -+ return 0; -+#endif -+ } -+ -+ unregister_chrdev_region(dev, 1); -+ } -+ -+ return err; -+} -+ -+ -+ -+/* -+ * Terminate the UMP device driver -+ */ -+void ump_kernel_device_terminate(void) -+{ -+ dev_t dev = MKDEV(ump_major, 0); -+ -+#if UMP_LICENSE_IS_GPL -+ device_destroy(ump_device.ump_class, dev); -+ class_destroy(ump_device.ump_class); -+#endif -+ -+ /* unregister char device */ -+ cdev_del(&ump_device.cdev); -+ -+ /* free major */ -+ unregister_chrdev_region(dev, 1); -+ -+#if UMP_LICENSE_IS_GPL -+ if (ump_debugfs_dir) -+ debugfs_remove_recursive(ump_debugfs_dir); -+#endif -+} -+ -+/* -+ * Open a new session. User space has called open() on us. -+ */ -+static int ump_file_open(struct inode *inode, struct file *filp) -+{ -+ struct ump_session_data *session_data; -+ _mali_osk_errcode_t err; -+ -+ /* input validation */ -+ if (0 != MINOR(inode->i_rdev)) { -+ MSG_ERR(("Minor not zero in ump_file_open()\n")); -+ return -ENODEV; -+ } -+ -+ /* Call the OS-Independent UMP Open function */ -+ err = _ump_ukk_open((void **) &session_data); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("Ump failed to open a new session\n")); -+ return ump_map_errcode(err); -+ } -+ -+ filp->private_data = (void *)session_data; -+ filp->f_pos = 0; -+ -+ return 0; /* success */ -+} -+ -+ -+ -+/* -+ * Close a session. User space has called close() or crashed/terminated. -+ */ -+static int ump_file_release(struct inode *inode, struct file *filp) -+{ -+ _mali_osk_errcode_t err; -+ -+ err = _ump_ukk_close((void **) &filp->private_data); -+ if (_MALI_OSK_ERR_OK != err) { -+ return ump_map_errcode(err); -+ } -+ -+ return 0; /* success */ -+} -+ -+ -+ -+/* -+ * Handle IOCTL requests. -+ */ -+#ifdef HAVE_UNLOCKED_IOCTL -+static long ump_file_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -+#else -+static int ump_file_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg) -+#endif -+{ -+ int err = -ENOTTY; -+ void __user *argument; -+ struct ump_session_data *session_data; -+ -+#ifndef HAVE_UNLOCKED_IOCTL -+ (void)inode; /* inode not used */ -+#endif -+ -+ session_data = (struct ump_session_data *)filp->private_data; -+ if (NULL == session_data) { -+ MSG_ERR(("No session data attached to file object\n")); -+ return -ENOTTY; -+ } -+ -+ /* interpret the argument as a user pointer to something */ -+ argument = (void __user *)arg; -+ -+ switch (cmd) { -+ case UMP_IOC_QUERY_API_VERSION: -+ err = ump_get_api_version_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_ALLOCATE : -+ err = ump_allocate_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_RELEASE: -+ err = ump_release_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_SIZE_GET: -+ err = ump_size_get_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_MSYNC: -+ err = ump_msync_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_CACHE_OPERATIONS_CONTROL: -+ err = ump_cache_operations_control_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_SWITCH_HW_USAGE: -+ err = ump_switch_hw_usage_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_LOCK: -+ err = ump_lock_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_UNLOCK: -+ err = ump_unlock_wrapper((u32 __user *)argument, session_data); -+ break; -+ -+ case UMP_IOC_DMABUF_IMPORT: -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ err = ump_dmabuf_import_wrapper((u32 __user *)argument, session_data); -+#else -+ err = -EFAULT; -+ DBG_MSG(1, ("User space use dmabuf API, but kernel don't support DMA BUF\n")); -+#endif -+ break; -+ -+ default: -+ DBG_MSG(1, ("No handler for IOCTL. cmd: 0x%08x, arg: 0x%08lx\n", cmd, arg)); -+ err = -EFAULT; -+ break; -+ } -+ -+ return err; -+} -+ -+int ump_map_errcode(_mali_osk_errcode_t err) -+{ -+ switch (err) { -+ case _MALI_OSK_ERR_OK : -+ return 0; -+ case _MALI_OSK_ERR_FAULT: -+ return -EFAULT; -+ case _MALI_OSK_ERR_INVALID_FUNC: -+ return -ENOTTY; -+ case _MALI_OSK_ERR_INVALID_ARGS: -+ return -EINVAL; -+ case _MALI_OSK_ERR_NOMEM: -+ return -ENOMEM; -+ case _MALI_OSK_ERR_TIMEOUT: -+ return -ETIMEDOUT; -+ case _MALI_OSK_ERR_RESTARTSYSCALL: -+ return -ERESTARTSYS; -+ case _MALI_OSK_ERR_ITEM_NOT_FOUND: -+ return -ENOENT; -+ default: -+ return -EFAULT; -+ } -+} -+ -+/* -+ * Handle from OS to map specified virtual memory to specified UMP memory. -+ */ -+static int ump_file_mmap(struct file *filp, struct vm_area_struct *vma) -+{ -+ _ump_uk_map_mem_s args; -+ _mali_osk_errcode_t err; -+ struct ump_session_data *session_data; -+ -+ /* Validate the session data */ -+ session_data = (struct ump_session_data *)filp->private_data; -+ if (NULL == session_data) { -+ MSG_ERR(("mmap() called without any session data available\n")); -+ return -EFAULT; -+ } -+ -+ /* Re-pack the arguments that mmap() packed for us */ -+ args.ctx = session_data; -+ args.phys_addr = 0; -+ args.size = vma->vm_end - vma->vm_start; -+ args._ukk_private = vma; -+ args.secure_id = vma->vm_pgoff; -+ -+ /* By setting this flag, during a process fork; the child process will not have the parent UMP mappings */ -+ vma->vm_flags |= VM_DONTCOPY; -+ -+ DBG_MSG(4, ("UMP vma->flags: %x\n", vma->vm_flags)); -+ -+ /* Call the common mmap handler */ -+ err = _ump_ukk_map_mem(&args); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("_ump_ukk_map_mem() failed in function ump_file_mmap()")); -+ return ump_map_errcode(err); -+ } -+ -+ return 0; /* success */ -+} -+ -+/* Export UMP kernel space API functions */ -+EXPORT_SYMBOL(ump_dd_secure_id_get); -+EXPORT_SYMBOL(ump_dd_handle_create_from_secure_id); -+EXPORT_SYMBOL(ump_dd_phys_block_count_get); -+EXPORT_SYMBOL(ump_dd_phys_block_get); -+EXPORT_SYMBOL(ump_dd_phys_blocks_get); -+EXPORT_SYMBOL(ump_dd_size_get); -+EXPORT_SYMBOL(ump_dd_reference_add); -+EXPORT_SYMBOL(ump_dd_reference_release); -+ -+/* Export our own extended kernel space allocator */ -+EXPORT_SYMBOL(ump_dd_handle_create_from_phys_blocks); -+ -+/* Setup init and exit functions for this module */ -+module_init(ump_initialize_module); -+module_exit(ump_cleanup_module); -+ -+/* And some module informatio */ -+MODULE_LICENSE(UMP_KERNEL_LINUX_LICENSE); -+MODULE_AUTHOR("ARM Ltd."); -+MODULE_VERSION(SVN_REV_STRING); -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h -new file mode 100755 -index 000000000..8d32ddbb5 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_linux.h -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (C) 2010-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __UMP_KERNEL_LINUX_H__ -+#define __UMP_KERNEL_LINUX_H__ -+ -+int ump_kernel_device_initialize(void); -+void ump_kernel_device_terminate(void); -+ -+ -+#endif /* __UMP_KERNEL_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c -new file mode 100755 -index 000000000..5a1257a25 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.c -@@ -0,0 +1,271 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/* needed to detect kernel version specific code */ -+#include -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+#include -+#else /* pre 2.6.26 the file was in the arch specific location */ -+#include -+#endif -+ -+#include -+#include -+#include -+#include -+#include "ump_kernel_common.h" -+#include "ump_kernel_memory_backend.h" -+ -+ -+ -+#define UMP_BLOCK_SIZE (256UL * 1024UL) /* 256kB, remember to keep the ()s */ -+ -+ -+ -+typedef struct block_info { -+ struct block_info *next; -+} block_info; -+ -+ -+ -+typedef struct block_allocator { -+ struct semaphore mutex; -+ block_info *all_blocks; -+ block_info *first_free; -+ u32 base; -+ u32 num_blocks; -+ u32 num_free; -+} block_allocator; -+ -+ -+static void block_allocator_shutdown(ump_memory_backend *backend); -+static int block_allocator_allocate(void *ctx, ump_dd_mem *mem); -+static void block_allocator_release(void *ctx, ump_dd_mem *handle); -+static inline u32 get_phys(block_allocator *allocator, block_info *block); -+static u32 block_allocator_stat(struct ump_memory_backend *backend); -+ -+ -+ -+/* -+ * Create dedicated memory backend -+ */ -+ump_memory_backend *ump_block_allocator_create(u32 base_address, u32 size) -+{ -+ ump_memory_backend *backend; -+ block_allocator *allocator; -+ u32 usable_size; -+ u32 num_blocks; -+ -+ usable_size = (size + UMP_BLOCK_SIZE - 1) & ~(UMP_BLOCK_SIZE - 1); -+ num_blocks = usable_size / UMP_BLOCK_SIZE; -+ -+ if (0 == usable_size) { -+ DBG_MSG(1, ("Memory block of size %u is unusable\n", size)); -+ return NULL; -+ } -+ -+ DBG_MSG(5, ("Creating dedicated UMP memory backend. Base address: 0x%08x, size: 0x%08x\n", base_address, size)); -+ DBG_MSG(6, ("%u usable bytes which becomes %u blocks\n", usable_size, num_blocks)); -+ -+ backend = kzalloc(sizeof(ump_memory_backend), GFP_KERNEL); -+ if (NULL != backend) { -+ allocator = kmalloc(sizeof(block_allocator), GFP_KERNEL); -+ if (NULL != allocator) { -+ allocator->all_blocks = kmalloc(sizeof(block_info) * num_blocks, GFP_KERNEL); -+ if (NULL != allocator->all_blocks) { -+ int i; -+ -+ allocator->first_free = NULL; -+ allocator->num_blocks = num_blocks; -+ allocator->num_free = num_blocks; -+ allocator->base = base_address; -+ sema_init(&allocator->mutex, 1); -+ -+ for (i = 0; i < num_blocks; i++) { -+ allocator->all_blocks[i].next = allocator->first_free; -+ allocator->first_free = &allocator->all_blocks[i]; -+ } -+ -+ backend->ctx = allocator; -+ backend->allocate = block_allocator_allocate; -+ backend->release = block_allocator_release; -+ backend->shutdown = block_allocator_shutdown; -+ backend->stat = block_allocator_stat; -+ backend->pre_allocate_physical_check = NULL; -+ backend->adjust_to_mali_phys = NULL; -+ -+ return backend; -+ } -+ kfree(allocator); -+ } -+ kfree(backend); -+ } -+ -+ return NULL; -+} -+ -+ -+ -+/* -+ * Destroy specified dedicated memory backend -+ */ -+static void block_allocator_shutdown(ump_memory_backend *backend) -+{ -+ block_allocator *allocator; -+ -+ BUG_ON(!backend); -+ BUG_ON(!backend->ctx); -+ -+ allocator = (block_allocator *)backend->ctx; -+ -+ DBG_MSG_IF(1, allocator->num_free != allocator->num_blocks, ("%u blocks still in use during shutdown\n", allocator->num_blocks - allocator->num_free)); -+ -+ kfree(allocator->all_blocks); -+ kfree(allocator); -+ kfree(backend); -+} -+ -+ -+ -+static int block_allocator_allocate(void *ctx, ump_dd_mem *mem) -+{ -+ block_allocator *allocator; -+ u32 left; -+ block_info *last_allocated = NULL; -+ int i = 0; -+ -+ BUG_ON(!ctx); -+ BUG_ON(!mem); -+ -+ allocator = (block_allocator *)ctx; -+ left = mem->size_bytes; -+ -+ BUG_ON(!left); -+ BUG_ON(!&allocator->mutex); -+ -+ mem->nr_blocks = ((left + UMP_BLOCK_SIZE - 1) & ~(UMP_BLOCK_SIZE - 1)) / UMP_BLOCK_SIZE; -+ mem->block_array = (ump_dd_physical_block *)vmalloc(sizeof(ump_dd_physical_block) * mem->nr_blocks); -+ if (NULL == mem->block_array) { -+ MSG_ERR(("Failed to allocate block array\n")); -+ return 0; -+ } -+ -+ if (down_interruptible(&allocator->mutex)) { -+ MSG_ERR(("Could not get mutex to do block_allocate\n")); -+ return 0; -+ } -+ -+ mem->size_bytes = 0; -+ -+ while ((left > 0) && (allocator->first_free)) { -+ block_info *block; -+ -+ block = allocator->first_free; -+ allocator->first_free = allocator->first_free->next; -+ block->next = last_allocated; -+ last_allocated = block; -+ allocator->num_free--; -+ -+ mem->block_array[i].addr = get_phys(allocator, block); -+ mem->block_array[i].size = UMP_BLOCK_SIZE; -+ mem->size_bytes += UMP_BLOCK_SIZE; -+ -+ i++; -+ -+ if (left < UMP_BLOCK_SIZE) left = 0; -+ else left -= UMP_BLOCK_SIZE; -+ } -+ -+ if (left) { -+ block_info *block; -+ /* release all memory back to the pool */ -+ while (last_allocated) { -+ block = last_allocated->next; -+ last_allocated->next = allocator->first_free; -+ allocator->first_free = last_allocated; -+ last_allocated = block; -+ allocator->num_free++; -+ } -+ -+ vfree(mem->block_array); -+ mem->backend_info = NULL; -+ mem->block_array = NULL; -+ -+ DBG_MSG(4, ("Could not find a mem-block for the allocation.\n")); -+ up(&allocator->mutex); -+ -+ return 0; -+ } -+ -+ mem->backend_info = last_allocated; -+ -+ up(&allocator->mutex); -+ mem->is_cached = 0; -+ -+ return 1; -+} -+ -+ -+ -+static void block_allocator_release(void *ctx, ump_dd_mem *handle) -+{ -+ block_allocator *allocator; -+ block_info *block, * next; -+ -+ BUG_ON(!ctx); -+ BUG_ON(!handle); -+ -+ allocator = (block_allocator *)ctx; -+ block = (block_info *)handle->backend_info; -+ BUG_ON(!block); -+ -+ if (down_interruptible(&allocator->mutex)) { -+ MSG_ERR(("Allocator release: Failed to get mutex - memory leak\n")); -+ return; -+ } -+ -+ while (block) { -+ next = block->next; -+ -+ BUG_ON((block < allocator->all_blocks) || (block > (allocator->all_blocks + allocator->num_blocks))); -+ -+ block->next = allocator->first_free; -+ allocator->first_free = block; -+ allocator->num_free++; -+ -+ block = next; -+ } -+ DBG_MSG(3, ("%d blocks free after release call\n", allocator->num_free)); -+ up(&allocator->mutex); -+ -+ vfree(handle->block_array); -+ handle->block_array = NULL; -+} -+ -+ -+ -+/* -+ * Helper function for calculating the physical base adderss of a memory block -+ */ -+static inline u32 get_phys(block_allocator *allocator, block_info *block) -+{ -+ return allocator->base + ((block - allocator->all_blocks) * UMP_BLOCK_SIZE); -+} -+ -+static u32 block_allocator_stat(struct ump_memory_backend *backend) -+{ -+ block_allocator *allocator; -+ BUG_ON(!backend); -+ allocator = (block_allocator *)backend->ctx; -+ BUG_ON(!allocator); -+ -+ return (allocator->num_blocks - allocator->num_free) * UMP_BLOCK_SIZE; -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h -new file mode 100755 -index 000000000..949fd245c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_dedicated.h -@@ -0,0 +1,23 @@ -+/* -+ * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_kernel_memory_backend_dedicated.h -+ */ -+ -+#ifndef __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ -+#define __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ -+ -+#include "ump_kernel_memory_backend.h" -+ -+ump_memory_backend *ump_block_allocator_create(u32 base_address, u32 size); -+ -+#endif /* __UMP_KERNEL_MEMORY_BACKEND_DEDICATED_H__ */ -+ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c -new file mode 100755 -index 000000000..7cd8d5d38 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.c -@@ -0,0 +1,235 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/* needed to detect kernel version specific code */ -+#include -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+#include -+#else /* pre 2.6.26 the file was in the arch specific location */ -+#include -+#endif -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include "ump_kernel_common.h" -+#include "ump_kernel_memory_backend.h" -+ -+ -+ -+typedef struct os_allocator { -+ struct semaphore mutex; -+ u32 num_pages_max; /**< Maximum number of pages to allocate from the OS */ -+ u32 num_pages_allocated; /**< Number of pages allocated from the OS */ -+} os_allocator; -+ -+ -+ -+static void os_free(void *ctx, ump_dd_mem *descriptor); -+static int os_allocate(void *ctx, ump_dd_mem *descriptor); -+static void os_memory_backend_destroy(ump_memory_backend *backend); -+static u32 os_stat(struct ump_memory_backend *backend); -+ -+ -+ -+/* -+ * Create OS memory backend -+ */ -+ump_memory_backend *ump_os_memory_backend_create(const int max_allocation) -+{ -+ ump_memory_backend *backend; -+ os_allocator *info; -+ -+ info = kmalloc(sizeof(os_allocator), GFP_KERNEL); -+ if (NULL == info) { -+ return NULL; -+ } -+ -+ info->num_pages_max = max_allocation >> PAGE_SHIFT; -+ info->num_pages_allocated = 0; -+ -+ sema_init(&info->mutex, 1); -+ -+ backend = kmalloc(sizeof(ump_memory_backend), GFP_KERNEL); -+ if (NULL == backend) { -+ kfree(info); -+ return NULL; -+ } -+ -+ backend->ctx = info; -+ backend->allocate = os_allocate; -+ backend->release = os_free; -+ backend->shutdown = os_memory_backend_destroy; -+ backend->stat = os_stat; -+ backend->pre_allocate_physical_check = NULL; -+ backend->adjust_to_mali_phys = NULL; -+ -+ return backend; -+} -+ -+ -+ -+/* -+ * Destroy specified OS memory backend -+ */ -+static void os_memory_backend_destroy(ump_memory_backend *backend) -+{ -+ os_allocator *info = (os_allocator *)backend->ctx; -+ -+ DBG_MSG_IF(1, 0 != info->num_pages_allocated, ("%d pages still in use during shutdown\n", info->num_pages_allocated)); -+ -+ kfree(info); -+ kfree(backend); -+} -+ -+ -+ -+/* -+ * Allocate UMP memory -+ */ -+static int os_allocate(void *ctx, ump_dd_mem *descriptor) -+{ -+ u32 left; -+ os_allocator *info; -+ int pages_allocated = 0; -+ int is_cached; -+ -+ BUG_ON(!descriptor); -+ BUG_ON(!ctx); -+ -+ info = (os_allocator *)ctx; -+ left = descriptor->size_bytes; -+ is_cached = descriptor->is_cached; -+ -+ if (down_interruptible(&info->mutex)) { -+ DBG_MSG(1, ("Failed to get mutex in os_free\n")); -+ return 0; /* failure */ -+ } -+ -+ descriptor->backend_info = NULL; -+ descriptor->nr_blocks = ((left + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) >> PAGE_SHIFT; -+ -+ DBG_MSG(5, ("Allocating page array. Size: %lu\n", descriptor->nr_blocks * sizeof(ump_dd_physical_block))); -+ -+ descriptor->block_array = (ump_dd_physical_block *)vmalloc(sizeof(ump_dd_physical_block) * descriptor->nr_blocks); -+ if (NULL == descriptor->block_array) { -+ up(&info->mutex); -+ DBG_MSG(1, ("Block array could not be allocated\n")); -+ return 0; /* failure */ -+ } -+ -+ while (left > 0 && ((info->num_pages_allocated + pages_allocated) < info->num_pages_max)) { -+ struct page *new_page; -+ -+ if (is_cached) { -+ new_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN); -+ } else { -+ new_page = alloc_page(GFP_HIGHUSER | __GFP_ZERO | __GFP_REPEAT | __GFP_NOWARN | __GFP_COLD); -+ } -+ if (NULL == new_page) { -+ break; -+ } -+ -+ /* Ensure page caches are flushed. */ -+ if (is_cached) { -+ descriptor->block_array[pages_allocated].addr = page_to_phys(new_page); -+ descriptor->block_array[pages_allocated].size = PAGE_SIZE; -+ } else { -+ descriptor->block_array[pages_allocated].addr = dma_map_page(NULL, new_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); -+ descriptor->block_array[pages_allocated].size = PAGE_SIZE; -+ } -+ -+ DBG_MSG(5, ("Allocated page 0x%08lx cached: %d\n", descriptor->block_array[pages_allocated].addr, is_cached)); -+ -+ if (left < PAGE_SIZE) { -+ left = 0; -+ } else { -+ left -= PAGE_SIZE; -+ } -+ -+ pages_allocated++; -+ } -+ -+ DBG_MSG(5, ("Alloce for ID:%2d got %d pages, cached: %d\n", descriptor->secure_id, pages_allocated)); -+ -+ if (left) { -+ DBG_MSG(1, ("Failed to allocate needed pages\n")); -+ -+ while (pages_allocated) { -+ pages_allocated--; -+ if (!is_cached) { -+ dma_unmap_page(NULL, descriptor->block_array[pages_allocated].addr, PAGE_SIZE, DMA_BIDIRECTIONAL); -+ } -+ __free_page(pfn_to_page(descriptor->block_array[pages_allocated].addr >> PAGE_SHIFT)); -+ } -+ -+ up(&info->mutex); -+ -+ return 0; /* failure */ -+ } -+ -+ info->num_pages_allocated += pages_allocated; -+ -+ DBG_MSG(6, ("%d out of %d pages now allocated\n", info->num_pages_allocated, info->num_pages_max)); -+ -+ up(&info->mutex); -+ -+ return 1; /* success*/ -+} -+ -+ -+/* -+ * Free specified UMP memory -+ */ -+static void os_free(void *ctx, ump_dd_mem *descriptor) -+{ -+ os_allocator *info; -+ int i; -+ -+ BUG_ON(!ctx); -+ BUG_ON(!descriptor); -+ -+ info = (os_allocator *)ctx; -+ -+ BUG_ON(descriptor->nr_blocks > info->num_pages_allocated); -+ -+ if (down_interruptible(&info->mutex)) { -+ DBG_MSG(1, ("Failed to get mutex in os_free\n")); -+ return; -+ } -+ -+ DBG_MSG(5, ("Releasing %lu OS pages\n", descriptor->nr_blocks)); -+ -+ info->num_pages_allocated -= descriptor->nr_blocks; -+ -+ up(&info->mutex); -+ -+ for (i = 0; i < descriptor->nr_blocks; i++) { -+ DBG_MSG(6, ("Freeing physical page. Address: 0x%08lx\n", descriptor->block_array[i].addr)); -+ if (! descriptor->is_cached) { -+ dma_unmap_page(NULL, descriptor->block_array[i].addr, PAGE_SIZE, DMA_BIDIRECTIONAL); -+ } -+ __free_page(pfn_to_page(descriptor->block_array[i].addr >> PAGE_SHIFT)); -+ } -+ -+ vfree(descriptor->block_array); -+} -+ -+ -+static u32 os_stat(struct ump_memory_backend *backend) -+{ -+ os_allocator *info; -+ info = (os_allocator *)backend->ctx; -+ return info->num_pages_allocated * _MALI_OSK_MALI_PAGE_SIZE; -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h -new file mode 100755 -index 000000000..d21d50351 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_memory_backend_os.h -@@ -0,0 +1,23 @@ -+/* -+ * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_kernel_memory_backend_os.h -+ */ -+ -+#ifndef __UMP_KERNEL_MEMORY_BACKEND_OS_H__ -+#define __UMP_KERNEL_MEMORY_BACKEND_OS_H__ -+ -+#include "ump_kernel_memory_backend.h" -+ -+ump_memory_backend *ump_os_memory_backend_create(const int max_allocation); -+ -+#endif /* __UMP_KERNEL_MEMORY_BACKEND_OS_H__ */ -+ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c -new file mode 100755 -index 000000000..6be0f8644 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.c -@@ -0,0 +1,222 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include "mali_kernel_common.h" -+#include "mali_osk.h" -+#include "ump_osk.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_types.h" -+#include "ump_kernel_random_mapping.h" -+ -+#include -+#include -+#include -+#include -+ -+ -+static ump_dd_mem *search(struct rb_root *root, int id) -+{ -+ struct rb_node *node = root->rb_node; -+ -+ while (node) { -+ ump_dd_mem *e = container_of(node, ump_dd_mem, node); -+ -+ if (id < e->secure_id) { -+ node = node->rb_left; -+ } else if (id > e->secure_id) { -+ node = node->rb_right; -+ } else { -+ return e; -+ } -+ } -+ -+ return NULL; -+} -+ -+static mali_bool insert(struct rb_root *root, int id, ump_dd_mem *mem) -+{ -+ struct rb_node **new = &(root->rb_node); -+ struct rb_node *parent = NULL; -+ -+ while (*new) { -+ ump_dd_mem *this = container_of(*new, ump_dd_mem, node); -+ -+ parent = *new; -+ if (id < this->secure_id) { -+ new = &((*new)->rb_left); -+ } else if (id > this->secure_id) { -+ new = &((*new)->rb_right); -+ } else { -+ printk(KERN_ERR "UMP: ID already used %x\n", id); -+ return MALI_FALSE; -+ } -+ } -+ -+ rb_link_node(&mem->node, parent, new); -+ rb_insert_color(&mem->node, root); -+ -+ return MALI_TRUE; -+} -+ -+ -+ump_random_mapping *ump_random_mapping_create(void) -+{ -+ ump_random_mapping *map = _mali_osk_calloc(1, sizeof(ump_random_mapping)); -+ -+ if (NULL == map) -+ return NULL; -+ -+ map->lock = _mali_osk_mutex_rw_init(_MALI_OSK_LOCKFLAG_ORDERED, -+ _MALI_OSK_LOCK_ORDER_DESCRIPTOR_MAP); -+ if (NULL != map->lock) { -+ map->root = RB_ROOT; -+#if UMP_RANDOM_MAP_DELAY -+ map->failed.count = 0; -+ map->failed.timestamp = jiffies; -+#endif -+ return map; -+ } -+ return NULL; -+} -+ -+void ump_random_mapping_destroy(ump_random_mapping *map) -+{ -+ _mali_osk_mutex_rw_term(map->lock); -+ _mali_osk_free(map); -+} -+ -+int ump_random_mapping_insert(ump_random_mapping *map, ump_dd_mem *mem) -+{ -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); -+ -+ while (1) { -+ u32 id; -+ -+ get_random_bytes(&id, sizeof(id)); -+ -+ /* Try a new random number if id happened to be the invalid -+ * secure ID (-1). */ -+ if (unlikely(id == UMP_INVALID_SECURE_ID)) -+ continue; -+ -+ /* Insert into the tree. If the id was already in use, get a -+ * new random id and try again. */ -+ if (insert(&map->root, id, mem)) { -+ mem->secure_id = id; -+ break; -+ } -+ } -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); -+ -+ return 0; -+} -+ -+ump_dd_mem *ump_random_mapping_get(ump_random_mapping *map, int id) -+{ -+ ump_dd_mem *mem = NULL; -+#if UMP_RANDOM_MAP_DELAY -+ int do_delay = 0; -+#endif -+ -+ DEBUG_ASSERT(map); -+ -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RO); -+ mem = search(&map->root, id); -+ -+ if (unlikely(NULL == mem)) { -+#if UMP_RANDOM_MAP_DELAY -+ map->failed.count++; -+ -+ if (time_is_before_jiffies(map->failed.timestamp + -+ UMP_FAILED_LOOKUP_DELAY * HZ)) { -+ /* If it is a long time since last failure, reset -+ * the counter and skip the delay this time. */ -+ map->failed.count = 0; -+ } else if (map->failed.count > UMP_FAILED_LOOKUPS_ALLOWED) { -+ do_delay = 1; -+ } -+ -+ map->failed.timestamp = jiffies; -+#endif /* UMP_RANDOM_MAP_DELAY */ -+ } else { -+ ump_dd_reference_add(mem); -+ } -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RO); -+ -+#if UMP_RANDOM_MAP_DELAY -+ if (do_delay) { -+ /* Apply delay */ -+ schedule_timeout_killable(UMP_FAILED_LOOKUP_DELAY); -+ } -+#endif /* UMP_RANDOM_MAP_DELAY */ -+ -+ return mem; -+} -+ -+static ump_dd_mem *ump_random_mapping_remove_internal(ump_random_mapping *map, int id) -+{ -+ ump_dd_mem *mem = NULL; -+ -+ mem = search(&map->root, id); -+ -+ if (mem) { -+ rb_erase(&mem->node, &map->root); -+ } -+ -+ return mem; -+} -+ -+void ump_random_mapping_put(ump_dd_mem *mem) -+{ -+ int new_ref; -+ -+ _mali_osk_mutex_rw_wait(device.secure_id_map->lock, _MALI_OSK_LOCKMODE_RW); -+ -+ new_ref = _ump_osk_atomic_dec_and_read(&mem->ref_count); -+ DBG_MSG(5, ("Memory reference decremented. ID: %u, new value: %d\n", -+ mem->secure_id, new_ref)); -+ -+ if (0 == new_ref) { -+ DBG_MSG(3, ("Final release of memory. ID: %u\n", mem->secure_id)); -+ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ if (mem->import_attach) { -+ struct dma_buf_attachment *attach = mem->import_attach; -+ struct dma_buf *dma_buf; -+ -+ if (mem->sgt) -+ dma_buf_unmap_attachment(attach, mem->sgt, -+ DMA_BIDIRECTIONAL); -+ -+ dma_buf = attach->dmabuf; -+ dma_buf_detach(attach->dmabuf, attach); -+ dma_buf_put(dma_buf); -+ -+ } -+#endif -+ ump_random_mapping_remove_internal(device.secure_id_map, mem->secure_id); -+ -+ mem->release_func(mem->ctx, mem); -+ _mali_osk_free(mem); -+ } -+ -+ _mali_osk_mutex_rw_signal(device.secure_id_map->lock, _MALI_OSK_LOCKMODE_RW); -+} -+ -+ump_dd_mem *ump_random_mapping_remove(ump_random_mapping *map, int descriptor) -+{ -+ ump_dd_mem *mem; -+ -+ _mali_osk_mutex_rw_wait(map->lock, _MALI_OSK_LOCKMODE_RW); -+ mem = ump_random_mapping_remove_internal(map, descriptor); -+ _mali_osk_mutex_rw_signal(map->lock, _MALI_OSK_LOCKMODE_RW); -+ -+ return mem; -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h -new file mode 100755 -index 000000000..2cea6cedc ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_kernel_random_mapping.h -@@ -0,0 +1,84 @@ -+/* -+ * Copyright (C) 2010-2011, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_kernel_random_mapping.h -+ */ -+ -+#ifndef __UMP_KERNEL_RANDOM_MAPPING_H__ -+#define __UMP_KERNEL_RANDOM_MAPPING_H__ -+ -+#include "mali_osk.h" -+#include -+ -+#define UMP_RANDOM_MAP_DELAY 1 -+#define UMP_FAILED_LOOKUP_DELAY 10 /* ms */ -+#define UMP_FAILED_LOOKUPS_ALLOWED 10 /* number of allowed failed lookups */ -+ -+/** -+ * The random mapping object -+ * Provides a separate namespace where we can map an integer to a pointer -+ */ -+typedef struct ump_random_mapping { -+ _mali_osk_mutex_rw_t *lock; /**< Lock protecting access to the mapping object */ -+ struct rb_root root; -+#if UMP_RANDOM_MAP_DELAY -+ struct { -+ unsigned long count; -+ unsigned long timestamp; -+ } failed; -+#endif -+} ump_random_mapping; -+ -+/** -+ * Create a random mapping object -+ * Create a random mapping capable of holding 2^20 entries -+ * @return Pointer to a random mapping object, NULL on failure -+ */ -+ump_random_mapping *ump_random_mapping_create(void); -+ -+/** -+ * Destroy a random mapping object -+ * @param map The map to free -+ */ -+void ump_random_mapping_destroy(ump_random_mapping *map); -+ -+/** -+ * Allocate a new mapping entry (random ID) -+ * Allocates a new entry in the map. -+ * @param map The map to allocate a new entry in -+ * @param target The value to map to -+ * @return The random allocated, a negative value on error -+ */ -+int ump_random_mapping_insert(ump_random_mapping *map, ump_dd_mem *mem); -+ -+/** -+ * Get the value mapped to by a random ID -+ * -+ * If the lookup fails, punish the calling thread by applying a delay. -+ * -+ * @param map The map to lookup the random id in -+ * @param id The ID to lookup -+ * @param target Pointer to a pointer which will receive the stored value -+ * @return ump_dd_mem pointer on successful lookup, NULL on error -+ */ -+ump_dd_mem *ump_random_mapping_get(ump_random_mapping *map, int id); -+ -+void ump_random_mapping_put(ump_dd_mem *mem); -+ -+/** -+ * Free the random ID -+ * For the random to be reused it has to be freed -+ * @param map The map to free the random from -+ * @param id The ID to free -+ */ -+ump_dd_mem *ump_random_mapping_remove(ump_random_mapping *map, int id); -+ -+#endif /* __UMP_KERNEL_RANDOM_MAPPING_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c -new file mode 100755 -index 000000000..e41931e1e ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_memory_backend.c -@@ -0,0 +1,65 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include /* kernel module definitions */ -+#include /* request_mem_region */ -+ -+#include "arch/config.h" /* Configuration for current platform. The symlink for arch is set by Makefile */ -+ -+#include "ump_osk.h" -+#include "ump_kernel_common.h" -+#include "ump_kernel_memory_backend_os.h" -+#include "ump_kernel_memory_backend_dedicated.h" -+ -+/* Configure which dynamic memory allocator to use */ -+int ump_backend = ARCH_UMP_BACKEND_DEFAULT; -+module_param(ump_backend, int, S_IRUGO); /* r--r--r-- */ -+MODULE_PARM_DESC(ump_backend, "0 = dedicated memory backend (default), 1 = OS memory backend"); -+ -+/* The base address of the memory block for the dedicated memory backend */ -+unsigned int ump_memory_address = ARCH_UMP_MEMORY_ADDRESS_DEFAULT; -+module_param(ump_memory_address, uint, S_IRUGO); /* r--r--r-- */ -+MODULE_PARM_DESC(ump_memory_address, "The physical address to map for the dedicated memory backend"); -+ -+/* The size of the memory block for the dedicated memory backend */ -+unsigned int ump_memory_size = ARCH_UMP_MEMORY_SIZE_DEFAULT; -+module_param(ump_memory_size, uint, S_IRUGO); /* r--r--r-- */ -+MODULE_PARM_DESC(ump_memory_size, "The size of fixed memory to map in the dedicated memory backend"); -+ -+ump_memory_backend *ump_memory_backend_create(void) -+{ -+ ump_memory_backend *backend = NULL; -+ -+ /* Create the dynamic memory allocator backend */ -+ if (0 == ump_backend) { -+ DBG_MSG(2, ("Using dedicated memory backend\n")); -+ -+ DBG_MSG(2, ("Requesting dedicated memory: 0x%08x, size: %u\n", ump_memory_address, ump_memory_size)); -+ /* Ask the OS if we can use the specified physical memory */ -+ if (NULL == request_mem_region(ump_memory_address, ump_memory_size, "UMP Memory")) { -+ MSG_ERR(("Failed to request memory region (0x%08X - 0x%08X). Is Mali DD already loaded?\n", ump_memory_address, ump_memory_address + ump_memory_size - 1)); -+ return NULL; -+ } -+ backend = ump_block_allocator_create(ump_memory_address, ump_memory_size); -+ } else if (1 == ump_backend) { -+ DBG_MSG(2, ("Using OS memory backend, allocation limit: %d\n", ump_memory_size)); -+ backend = ump_os_memory_backend_create(ump_memory_size); -+ } -+ -+ return backend; -+} -+ -+void ump_memory_backend_destroy(void) -+{ -+ if (0 == ump_backend) { -+ DBG_MSG(2, ("Releasing dedicated memory: 0x%08x\n", ump_memory_address)); -+ release_mem_region(ump_memory_address, ump_memory_size); -+ } -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c -new file mode 100755 -index 000000000..2b634ba79 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_atomics.c -@@ -0,0 +1,27 @@ -+/* -+ * Copyright (C) 2010, 2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_osk_atomics.c -+ * Implementation of the OS abstraction layer for the UMP kernel device driver -+ */ -+ -+#include "ump_osk.h" -+#include -+ -+int _ump_osk_atomic_dec_and_read(_mali_osk_atomic_t *atom) -+{ -+ return atomic_dec_return((atomic_t *)&atom->u.val); -+} -+ -+int _ump_osk_atomic_inc_and_read(_mali_osk_atomic_t *atom) -+{ -+ return atomic_inc_return((atomic_t *)&atom->u.val); -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c -new file mode 100755 -index 000000000..e08bf2525 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_low_level_mem.c -@@ -0,0 +1,314 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_osk_memory.c -+ * Implementation of the OS abstraction layer for the kernel device driver -+ */ -+ -+/* needed to detect kernel version specific code */ -+#include -+ -+#include "ump_osk.h" -+#include "ump_uk_types.h" -+#include "ump_ukk.h" -+#include "ump_kernel_common.h" -+#include /* kernel module definitions */ -+#include -+#include -+#include -+ -+#include -+#include /* to verify pointers from user space */ -+#include -+#include -+ -+typedef struct ump_vma_usage_tracker { -+ atomic_t references; -+ ump_memory_allocation *descriptor; -+} ump_vma_usage_tracker; -+ -+static void ump_vma_open(struct vm_area_struct *vma); -+static void ump_vma_close(struct vm_area_struct *vma); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+static int ump_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf); -+#else -+static unsigned long ump_cpu_page_fault_handler(struct vm_area_struct *vma, unsigned long address); -+#endif -+ -+static struct vm_operations_struct ump_vm_ops = { -+ .open = ump_vma_open, -+ .close = ump_vma_close, -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+ .fault = ump_cpu_page_fault_handler -+#else -+ .nopfn = ump_cpu_page_fault_handler -+#endif -+}; -+ -+/* -+ * Page fault for VMA region -+ * This should never happen since we always map in the entire virtual memory range. -+ */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+static int ump_cpu_page_fault_handler(struct vm_area_struct *vma, struct vm_fault *vmf) -+#else -+static unsigned long ump_cpu_page_fault_handler(struct vm_area_struct *vma, unsigned long address) -+#endif -+{ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+ void __user *address; -+ address = vmf->virtual_address; -+#endif -+ MSG_ERR(("Page-fault in UMP memory region caused by the CPU\n")); -+ MSG_ERR(("VMA: 0x%08lx, virtual address: 0x%08lx\n", (unsigned long)vma, address)); -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) -+ return VM_FAULT_SIGBUS; -+#else -+ return NOPFN_SIGBUS; -+#endif -+} -+ -+static void ump_vma_open(struct vm_area_struct *vma) -+{ -+ ump_vma_usage_tracker *vma_usage_tracker; -+ int new_val; -+ -+ vma_usage_tracker = (ump_vma_usage_tracker *)vma->vm_private_data; -+ BUG_ON(NULL == vma_usage_tracker); -+ -+ new_val = atomic_inc_return(&vma_usage_tracker->references); -+ -+ DBG_MSG(4, ("VMA open, VMA reference count incremented. VMA: 0x%08lx, reference count: %d\n", (unsigned long)vma, new_val)); -+} -+ -+static void ump_vma_close(struct vm_area_struct *vma) -+{ -+ ump_vma_usage_tracker *vma_usage_tracker; -+ _ump_uk_unmap_mem_s args; -+ int new_val; -+ -+ vma_usage_tracker = (ump_vma_usage_tracker *)vma->vm_private_data; -+ BUG_ON(NULL == vma_usage_tracker); -+ -+ new_val = atomic_dec_return(&vma_usage_tracker->references); -+ -+ DBG_MSG(4, ("VMA close, VMA reference count decremented. VMA: 0x%08lx, reference count: %d\n", (unsigned long)vma, new_val)); -+ -+ if (0 == new_val) { -+ ump_memory_allocation *descriptor; -+ -+ descriptor = vma_usage_tracker->descriptor; -+ -+ args.ctx = descriptor->ump_session; -+ args.cookie = descriptor->cookie; -+ args.mapping = descriptor->mapping; -+ args.size = descriptor->size; -+ -+ args._ukk_private = NULL; /** @note unused */ -+ -+ DBG_MSG(4, ("No more VMA references left, releasing UMP memory\n")); -+ _ump_ukk_unmap_mem(& args); -+ -+ /* vma_usage_tracker is free()d by _ump_osk_mem_mapregion_term() */ -+ } -+} -+ -+_mali_osk_errcode_t _ump_osk_mem_mapregion_init(ump_memory_allocation *descriptor) -+{ -+ ump_vma_usage_tracker *vma_usage_tracker; -+ struct vm_area_struct *vma; -+ -+ if (NULL == descriptor) return _MALI_OSK_ERR_FAULT; -+ -+ vma_usage_tracker = kmalloc(sizeof(ump_vma_usage_tracker), GFP_KERNEL); -+ if (NULL == vma_usage_tracker) { -+ DBG_MSG(1, ("Failed to allocate memory for ump_vma_usage_tracker in _mali_osk_mem_mapregion_init\n")); -+ return -_MALI_OSK_ERR_FAULT; -+ } -+ -+ vma = (struct vm_area_struct *)descriptor->process_mapping_info; -+ if (NULL == vma) { -+ kfree(vma_usage_tracker); -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ vma->vm_private_data = vma_usage_tracker; -+ vma->vm_flags |= VM_IO; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) -+ vma->vm_flags |= VM_RESERVED; -+#else -+ vma->vm_flags |= VM_DONTDUMP; -+ vma->vm_flags |= VM_DONTEXPAND; -+ vma->vm_flags |= VM_PFNMAP; -+#endif -+ -+ -+ if (0 == descriptor->is_cached) { -+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); -+ } -+ DBG_MSG(3, ("Mapping with page_prot: 0x%x\n", vma->vm_page_prot)); -+ -+ /* Setup the functions which handle further VMA handling */ -+ vma->vm_ops = &ump_vm_ops; -+ -+ /* Do the va range allocation - in this case, it was done earlier, so we copy in that information */ -+ descriptor->mapping = (void __user *)vma->vm_start; -+ -+ atomic_set(&vma_usage_tracker->references, 1); /*this can later be increased if process is forked, see ump_vma_open() */ -+ vma_usage_tracker->descriptor = descriptor; -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+void _ump_osk_mem_mapregion_term(ump_memory_allocation *descriptor) -+{ -+ struct vm_area_struct *vma; -+ ump_vma_usage_tracker *vma_usage_tracker; -+ -+ if (NULL == descriptor) return; -+ -+ /* Linux does the right thing as part of munmap to remove the mapping -+ * All that remains is that we remove the vma_usage_tracker setup in init() */ -+ vma = (struct vm_area_struct *)descriptor->process_mapping_info; -+ -+ vma_usage_tracker = vma->vm_private_data; -+ -+ /* We only get called if mem_mapregion_init succeeded */ -+ kfree(vma_usage_tracker); -+ return; -+} -+ -+_mali_osk_errcode_t _ump_osk_mem_mapregion_map(ump_memory_allocation *descriptor, u32 offset, u32 *phys_addr, unsigned long size) -+{ -+ struct vm_area_struct *vma; -+ _mali_osk_errcode_t retval; -+ -+ if (NULL == descriptor) return _MALI_OSK_ERR_FAULT; -+ -+ vma = (struct vm_area_struct *)descriptor->process_mapping_info; -+ -+ if (NULL == vma) return _MALI_OSK_ERR_FAULT; -+ -+ retval = remap_pfn_range(vma, ((u32)descriptor->mapping) + offset, (*phys_addr) >> PAGE_SHIFT, size, vma->vm_page_prot) ? _MALI_OSK_ERR_FAULT : _MALI_OSK_ERR_OK;; -+ -+ DBG_MSG(4, ("Mapping virtual to physical memory. ID: %u, vma: 0x%08lx, virtual addr:0x%08lx, physical addr: 0x%08lx, size:%lu, prot:0x%x, vm_flags:0x%x RETVAL: 0x%x\n", -+ ump_dd_secure_id_get(descriptor->handle), -+ (unsigned long)vma, -+ (unsigned long)(vma->vm_start + offset), -+ (unsigned long)*phys_addr, -+ size, -+ (unsigned int)vma->vm_page_prot, vma->vm_flags, retval)); -+ -+ return retval; -+} -+ -+static void level1_cache_flush_all(void) -+{ -+ DBG_MSG(4, ("UMP[xx] Flushing complete L1 cache\n")); -+ __cpuc_flush_kern_all(); -+} -+ -+void _ump_osk_msync(ump_dd_mem *mem, void *virt, u32 offset, u32 size, ump_uk_msync_op op, ump_session_data *session_data) -+{ -+ int i; -+ -+ /* Flush L1 using virtual address, the entire range in one go. -+ * Only flush if user space process has a valid write mapping on given address. */ -+ if ((mem) && (virt != NULL) && (access_ok(virt, size))) { -+ __cpuc_flush_dcache_area(virt, size); -+ DBG_MSG(3, ("UMP[%02u] Flushing CPU L1 Cache. CPU address: %x, size: %x\n", mem->secure_id, virt, size)); -+ } else { -+ if (session_data) { -+ if (op == _UMP_UK_MSYNC_FLUSH_L1) { -+ DBG_MSG(4, ("UMP Pending L1 cache flushes: %d\n", session_data->has_pending_level1_cache_flush)); -+ session_data->has_pending_level1_cache_flush = 0; -+ level1_cache_flush_all(); -+ return; -+ } else { -+ if (session_data->cache_operations_ongoing) { -+ session_data->has_pending_level1_cache_flush++; -+ DBG_MSG(4, ("UMP[%02u] Defering the L1 flush. Nr pending:%d\n", mem->secure_id, session_data->has_pending_level1_cache_flush)); -+ } else { -+ /* Flushing the L1 cache for each switch_user() if ump_cache_operations_control(START) is not called */ -+ level1_cache_flush_all(); -+ } -+ } -+ } else { -+ DBG_MSG(4, ("Unkown state %s %d\n", __FUNCTION__, __LINE__)); -+ level1_cache_flush_all(); -+ } -+ } -+ -+ if (NULL == mem) return; -+ -+ if (mem->size_bytes == size) { -+ DBG_MSG(3, ("UMP[%02u] Flushing CPU L2 Cache\n", mem->secure_id)); -+ } else { -+ DBG_MSG(3, ("UMP[%02u] Flushing CPU L2 Cache. Blocks:%u, TotalSize:%u. FlushSize:%u Offset:0x%x FirstPaddr:0x%08x\n", -+ mem->secure_id, mem->nr_blocks, mem->size_bytes, size, offset, mem->block_array[0].addr)); -+ } -+ -+ -+ /* Flush L2 using physical addresses, block for block. */ -+ for (i = 0 ; i < mem->nr_blocks; i++) { -+ u32 start_p, end_p; -+ ump_dd_physical_block *block; -+ block = &mem->block_array[i]; -+ -+ if (offset >= block->size) { -+ offset -= block->size; -+ continue; -+ } -+ -+ if (offset) { -+ start_p = (u32)block->addr + offset; -+ /* We'll zero the offset later, after using it to calculate end_p. */ -+ } else { -+ start_p = (u32)block->addr; -+ } -+ -+ if (size < block->size - offset) { -+ end_p = start_p + size; -+ size = 0; -+ } else { -+ if (offset) { -+ end_p = start_p + (block->size - offset); -+ size -= block->size - offset; -+ offset = 0; -+ } else { -+ end_p = start_p + block->size; -+ size -= block->size; -+ } -+ } -+ -+ switch (op) { -+ case _UMP_UK_MSYNC_CLEAN: -+ outer_clean_range(start_p, end_p); -+ break; -+ case _UMP_UK_MSYNC_CLEAN_AND_INVALIDATE: -+ outer_flush_range(start_p, end_p); -+ break; -+ case _UMP_UK_MSYNC_INVALIDATE: -+ outer_inv_range(start_p, end_p); -+ break; -+ default: -+ break; -+ } -+ -+ if (0 == size) { -+ /* Nothing left to flush. */ -+ break; -+ } -+ } -+ -+ return; -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c -new file mode 100755 -index 000000000..58c9f1bf2 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_osk_misc.c -@@ -0,0 +1,36 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_osk_misc.c -+ * Implementation of the OS abstraction layer for the UMP kernel device driver -+ */ -+ -+ -+#include "ump_osk.h" -+ -+#include -+#include "ump_kernel_linux.h" -+ -+/* is called from ump_kernel_constructor in common code */ -+_mali_osk_errcode_t _ump_osk_init(void) -+{ -+ if (0 != ump_kernel_device_initialize()) { -+ return _MALI_OSK_ERR_FAULT; -+ } -+ -+ return _MALI_OSK_ERR_OK; -+} -+ -+_mali_osk_errcode_t _ump_osk_term(void) -+{ -+ ump_kernel_device_terminate(); -+ return _MALI_OSK_ERR_OK; -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c -new file mode 100755 -index 000000000..56a787ff6 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.c -@@ -0,0 +1,230 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_ukk_wrappers.c -+ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls for the reference implementation -+ */ -+ -+ -+#include /* user space access */ -+ -+#include "ump_osk.h" -+#include "ump_uk_types.h" -+#include "ump_ukk.h" -+#include "ump_kernel_common.h" -+#include -+#include "ump_kernel_interface_ref_drv.h" -+#include "mali_osk_list.h" -+ -+extern struct device *ump_global_mdev; -+ -+/* -+ * IOCTL operation; Allocate UMP memory -+ */ -+int ump_allocate_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_allocate_s user_interaction; -+ _mali_osk_errcode_t err; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_allocate()\n")); -+ return -ENOTTY; -+ } -+ -+ /* Copy the user space memory to kernel space (so we safely can read it) */ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_allocate()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ -+ err = _ump_ukk_allocate(&user_interaction); -+ if (_MALI_OSK_ERR_OK != err) { -+ DBG_MSG(1, ("_ump_ukk_allocate() failed in ump_ioctl_allocate()\n")); -+ return ump_map_errcode(err); -+ } -+ user_interaction.ctx = NULL; -+ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ /* If the copy fails then we should release the memory. We can use the IOCTL release to accomplish this */ -+ _ump_uk_release_s release_args; -+ -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_allocate()\n")); -+ -+ release_args.ctx = (void *) session_data; -+ release_args.secure_id = user_interaction.secure_id; -+ -+ err = _ump_ukk_release(&release_args); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("_ump_ukk_release() also failed when trying to release newly allocated memory in ump_ioctl_allocate()\n")); -+ } -+ -+ return -EFAULT; -+ } -+ -+ return 0; /* success */ -+} -+ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+static ump_dd_handle get_ump_handle_from_dmabuf(struct ump_session_data *session_data, -+ struct dma_buf *dmabuf) -+{ -+ ump_session_memory_list_element *session_mem, *tmp; -+ struct dma_buf_attachment *attach; -+ ump_dd_handle ump_handle; -+ -+ DEBUG_ASSERT_POINTER(session_data); -+ -+ _mali_osk_mutex_wait(session_data->lock); -+ -+ _MALI_OSK_LIST_FOREACHENTRY(session_mem, tmp, -+ &session_data->list_head_session_memory_list, -+ ump_session_memory_list_element, list) { -+ if (session_mem->mem->import_attach) { -+ attach = session_mem->mem->import_attach; -+ if (attach->dmabuf == dmabuf) { -+ _mali_osk_mutex_signal(session_data->lock); -+ ump_handle = (ump_dd_handle)session_mem->mem; -+ ump_random_mapping_get(device.secure_id_map, ump_dd_secure_id_get(ump_handle)); -+ return ump_handle; -+ } -+ } -+ } -+ -+ _mali_osk_mutex_signal(session_data->lock); -+ -+ return NULL; -+} -+ -+int ump_dmabuf_import_wrapper(u32 __user *argument, -+ struct ump_session_data *session_data) -+{ -+ ump_session_memory_list_element *session = NULL; -+ _ump_uk_dmabuf_s ump_dmabuf; -+ ump_dd_handle ump_handle; -+ ump_dd_physical_block *blocks = NULL; -+ struct dma_buf_attachment *attach = NULL; -+ struct dma_buf *dma_buf; -+ struct sg_table *sgt = NULL; -+ struct scatterlist *sgl; -+ unsigned int i = 0; -+ int ret = 0; -+ -+ /* Sanity check input parameters */ -+ if (!argument || !session_data) { -+ MSG_ERR(("NULL parameter.\n")); -+ return -EINVAL; -+ } -+ -+ if (copy_from_user(&ump_dmabuf, argument, -+ sizeof(_ump_uk_dmabuf_s))) { -+ MSG_ERR(("copy_from_user() failed.\n")); -+ return -EFAULT; -+ } -+ -+ dma_buf = dma_buf_get(ump_dmabuf.fd); -+ if (IS_ERR(dma_buf)) -+ return PTR_ERR(dma_buf); -+ -+ /* -+ * if already imported then increase a refcount to the ump descriptor -+ * and call dma_buf_put() and then go to found to return previous -+ * ump secure id. -+ */ -+ ump_handle = get_ump_handle_from_dmabuf(session_data, dma_buf); -+ if (ump_handle) { -+ dma_buf_put(dma_buf); -+ goto found; -+ } -+ -+ attach = dma_buf_attach(dma_buf, ump_global_mdev); -+ if (IS_ERR(attach)) { -+ ret = PTR_ERR(attach); -+ goto err_dma_buf_put; -+ } -+ -+ sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); -+ if (IS_ERR(sgt)) { -+ ret = PTR_ERR(sgt); -+ goto err_dma_buf_detach; -+ } -+ -+ blocks = (ump_dd_physical_block *)_mali_osk_malloc(sizeof(ump_dd_physical_block) * sgt->nents); -+ if (!blocks) { -+ DBG_MSG(1, ("Failed to allocate blocks.\n")); -+ ret = -EFAULT; -+ goto err_dma_buf_unmap; -+ } -+ for_each_sg(sgt->sgl, sgl, sgt->nents, i) { -+ blocks[i].addr = sg_phys(sgl); -+ blocks[i].size = sg_dma_len(sgl); -+ } -+ -+ /* -+ * Initialize the session memory list element, and add it -+ * to the session object -+ */ -+ session = _mali_osk_calloc(1, sizeof(*session)); -+ if (!session) { -+ DBG_MSG(1, ("Failed to allocate session.\n")); -+ ret = -EFAULT; -+ goto err_free_block; -+ } -+ -+ ump_handle = ump_dd_handle_create_from_phys_blocks(blocks, i); -+ if (UMP_DD_HANDLE_INVALID == ump_handle) { -+ DBG_MSG(1, ("Failed to create ump handle.\n")); -+ ret = -EFAULT; -+ goto err_free_session; -+ } -+ -+ session->mem = (ump_dd_mem *)ump_handle; -+ session->mem->import_attach = attach; -+ session->mem->sgt = sgt; -+ -+ _mali_osk_mutex_wait(session_data->lock); -+ _mali_osk_list_add(&(session->list), -+ &(session_data->list_head_session_memory_list)); -+ _mali_osk_mutex_signal(session_data->lock); -+ -+ _mali_osk_free(blocks); -+ -+found: -+ ump_dmabuf.ctx = (void *)session_data; -+ ump_dmabuf.secure_id = ump_dd_secure_id_get(ump_handle); -+ ump_dmabuf.size = ump_dd_size_get(ump_handle); -+ -+ if (copy_to_user(argument, &ump_dmabuf, -+ sizeof(_ump_uk_dmabuf_s))) { -+ MSG_ERR(("copy_to_user() failed.\n")); -+ ret = -EFAULT; -+ goto err_release_ump_handle; -+ } -+ -+ return ret; -+ -+err_release_ump_handle: -+ ump_dd_reference_release(ump_handle); -+err_free_session: -+ _mali_osk_free(session); -+err_free_block: -+ _mali_osk_free(blocks); -+err_dma_buf_unmap: -+ dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); -+err_dma_buf_detach: -+ dma_buf_detach(dma_buf, attach); -+err_dma_buf_put: -+ dma_buf_put(dma_buf); -+ return ret; -+} -+#endif -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h -new file mode 100755 -index 000000000..61a7095a6 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_ref_wrappers.h -@@ -0,0 +1,36 @@ -+/* -+ * Copyright (C) 2010, 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_ukk_wrappers.h -+ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls for the reference implementation -+ */ -+ -+#ifndef __UMP_UKK_REF_WRAPPERS_H__ -+#define __UMP_UKK_REF_WRAPPERS_H__ -+ -+#include -+#include "ump_kernel_common.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+ -+int ump_allocate_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+#ifdef CONFIG_DMA_SHARED_BUFFER -+int ump_dmabuf_import_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+#endif -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __UMP_UKK_REF_WRAPPERS_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c -new file mode 100755 -index 000000000..4d6b69608 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.c -@@ -0,0 +1,280 @@ -+/* -+ * Copyright (C) 2010-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_ukk_wrappers.c -+ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls -+ */ -+ -+#include /* user space access */ -+ -+#include "ump_osk.h" -+#include "ump_uk_types.h" -+#include "ump_ukk.h" -+#include "ump_kernel_common.h" -+ -+/* -+ * IOCTL operation; Negotiate version of IOCTL API -+ */ -+int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_api_version_s version_info; -+ _mali_osk_errcode_t err; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_get_api_version()\n")); -+ return -ENOTTY; -+ } -+ -+ /* Copy the user space memory to kernel space (so we safely can read it) */ -+ if (0 != copy_from_user(&version_info, argument, sizeof(version_info))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n")); -+ return -EFAULT; -+ } -+ -+ version_info.ctx = (void *) session_data; -+ err = _ump_uku_get_api_version(&version_info); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("_ump_uku_get_api_version() failed in ump_ioctl_get_api_version()\n")); -+ return ump_map_errcode(err); -+ } -+ -+ version_info.ctx = NULL; -+ -+ /* Copy ouput data back to user space */ -+ if (0 != copy_to_user(argument, &version_info, sizeof(version_info))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_get_api_version()\n")); -+ return -EFAULT; -+ } -+ -+ return 0; /* success */ -+} -+ -+ -+/* -+ * IOCTL operation; Release reference to specified UMP memory. -+ */ -+int ump_release_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_release_s release_args; -+ _mali_osk_errcode_t err; -+ -+ /* Sanity check input parameters */ -+ if (NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_release()\n")); -+ return -ENOTTY; -+ } -+ -+ /* Copy the user space memory to kernel space (so we safely can read it) */ -+ if (0 != copy_from_user(&release_args, argument, sizeof(release_args))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_get_api_version()\n")); -+ return -EFAULT; -+ } -+ -+ release_args.ctx = (void *) session_data; -+ err = _ump_ukk_release(&release_args); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("_ump_ukk_release() failed in ump_ioctl_release()\n")); -+ return ump_map_errcode(err); -+ } -+ -+ -+ return 0; /* success */ -+} -+ -+/* -+ * IOCTL operation; Return size for specified UMP memory. -+ */ -+int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_size_get_s user_interaction; -+ _mali_osk_errcode_t err; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); -+ return -ENOTTY; -+ } -+ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_size_get()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ err = _ump_ukk_size_get(&user_interaction); -+ if (_MALI_OSK_ERR_OK != err) { -+ MSG_ERR(("_ump_ukk_size_get() failed in ump_ioctl_size_get()\n")); -+ return ump_map_errcode(err); -+ } -+ -+ user_interaction.ctx = NULL; -+ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_size_get()\n")); -+ return -EFAULT; -+ } -+ -+ return 0; /* success */ -+} -+ -+/* -+ * IOCTL operation; Do cache maintenance on specified UMP memory. -+ */ -+int ump_msync_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_msync_s user_interaction; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); -+ return -ENOTTY; -+ } -+ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_msync()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ -+ _ump_ukk_msync(&user_interaction); -+ -+ user_interaction.ctx = NULL; -+ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_msync()\n")); -+ return -EFAULT; -+ } -+ -+ return 0; /* success */ -+} -+int ump_cache_operations_control_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_cache_operations_control_s user_interaction; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); -+ return -ENOTTY; -+ } -+ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_cache_operations_control()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ -+ _ump_ukk_cache_operations_control((_ump_uk_cache_operations_control_s *) &user_interaction); -+ -+ user_interaction.ctx = NULL; -+ -+#if 0 /* No data to copy back */ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_cache_operations_control()\n")); -+ return -EFAULT; -+ } -+#endif -+ return 0; /* success */ -+} -+ -+int ump_switch_hw_usage_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_switch_hw_usage_s user_interaction; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); -+ return -ENOTTY; -+ } -+ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ -+ _ump_ukk_switch_hw_usage(&user_interaction); -+ -+ user_interaction.ctx = NULL; -+ -+#if 0 /* No data to copy back */ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); -+ return -EFAULT; -+ } -+#endif -+ return 0; /* success */ -+} -+ -+int ump_lock_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_lock_s user_interaction; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); -+ return -ENOTTY; -+ } -+ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ -+ _ump_ukk_lock(&user_interaction); -+ -+ user_interaction.ctx = NULL; -+ -+#if 0 /* No data to copy back */ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); -+ return -EFAULT; -+ } -+#endif -+ -+ return 0; /* success */ -+} -+ -+int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data *session_data) -+{ -+ _ump_uk_unlock_s user_interaction; -+ -+ /* Sanity check input parameters */ -+ if (NULL == argument || NULL == session_data) { -+ MSG_ERR(("NULL parameter in ump_ioctl_size_get()\n")); -+ return -ENOTTY; -+ } -+ -+ if (0 != copy_from_user(&user_interaction, argument, sizeof(user_interaction))) { -+ MSG_ERR(("copy_from_user() in ump_ioctl_switch_hw_usage()\n")); -+ return -EFAULT; -+ } -+ -+ user_interaction.ctx = (void *) session_data; -+ -+ _ump_ukk_unlock(&user_interaction); -+ -+ user_interaction.ctx = NULL; -+ -+#if 0 /* No data to copy back */ -+ if (0 != copy_to_user(argument, &user_interaction, sizeof(user_interaction))) { -+ MSG_ERR(("copy_to_user() failed in ump_ioctl_switch_hw_usage()\n")); -+ return -EFAULT; -+ } -+#endif -+ -+ return 0; /* success */ -+} -diff --git a/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h -new file mode 100755 -index 000000000..5f8fc683c ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/linux/ump_ukk_wrappers.h -@@ -0,0 +1,46 @@ -+/* -+ * Copyright (C) 2010, 2012-2014, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+/** -+ * @file ump_ukk_wrappers.h -+ * Defines the wrapper functions which turn Linux IOCTL calls into _ukk_ calls -+ */ -+ -+#ifndef __UMP_UKK_WRAPPERS_H__ -+#define __UMP_UKK_WRAPPERS_H__ -+ -+#include -+#include "ump_kernel_common.h" -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+ -+ -+int ump_get_api_version_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_release_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_size_get_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_msync_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_cache_operations_control_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_switch_hw_usage_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_lock_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+int ump_unlock_wrapper(u32 __user *argument, struct ump_session_data *session_data); -+ -+ -+ -+ -+#ifdef __cplusplus -+} -+#endif -+ -+ -+ -+#endif /* __UMP_UKK_WRAPPERS_H__ */ -diff --git a/drivers/gpu/arm/mali400/ump/readme.txt b/drivers/gpu/arm/mali400/ump/readme.txt -new file mode 100755 -index 000000000..c238cf0f2 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/ump/readme.txt -@@ -0,0 +1,28 @@ -+Building the UMP Device Driver for Linux -+---------------------------------------- -+ -+Build the UMP Device Driver for Linux by running the following make command: -+ -+KDIR= CONFIG= BUILD= make -+ -+where -+ kdir_path: Path to your Linux Kernel directory -+ your_config: Name of the sub-folder to find the required config.h file -+ ("arch-" will be prepended) -+ build_option: debug or release. Debug is default. -+ -+The config.h contains following configuration parameters: -+ -+ARCH_UMP_BACKEND_DEFAULT -+ 0 specifies the dedicated memory allocator. -+ 1 specifies the OS memory allocator. -+ARCH_UMP_MEMORY_ADDRESS_DEFAULT -+ This is only required for the dedicated memory allocator, and specifies -+ the physical start address of the memory block reserved for UMP. -+ARCH_UMP_MEMORY_SIZE_DEFAULT -+ This specified the size of the memory block reserved for UMP, or the -+ maximum limit for allocations from the OS. -+ -+The result will be a ump.ko file, which can be loaded into the Linux kernel -+by using the insmod command. The driver can also be built as a part of the -+kernel itself. -diff --git a/drivers/gpu/arm/mali400/umplock/Makefile b/drivers/gpu/arm/mali400/umplock/Makefile -new file mode 100755 -index 000000000..e5549a33f ---- /dev/null -+++ b/drivers/gpu/arm/mali400/umplock/Makefile -@@ -0,0 +1,69 @@ -+# -+# Copyright (C) 2012, 2016-2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+# as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained from Free Software -+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+# -+ -+# default to building for the host -+ARCH ?= $(shell uname -m) -+ -+# linux build system integration -+ -+ifneq ($(KERNELRELEASE),) -+# Inside the kernel build system -+ -+EXTRA_CFLAGS += -I$(KBUILD_EXTMOD) -+ -+SRC = umplock_driver.c -+ -+MODULE:=umplock.ko -+ -+obj-m := $(MODULE:.ko=.o) -+$(MODULE:.ko=-y) := $(SRC:.c=.o) -+ -+$(MODULE:.ko=-objs) := $(SRC:.c=.o) -+ -+else -+# Outside the kernel build system -+# -+# -+ -+# Get any user defined KDIR- or maybe even a hardcoded KDIR -+-include KDIR_CONFIGURATION -+ -+# Define host system directory -+KDIR-$(shell uname -m):=/lib/modules/$(shell uname -r)/build -+ -+ifeq ($(ARCH), arm) -+ # when compiling for ARM we're cross compiling -+ export CROSS_COMPILE ?= arm-none-linux-gnueabi- -+ CONFIG ?= arm -+else -+ # Compiling for the host -+ CONFIG ?= $(shell uname -m) -+endif -+ -+# default cpu to select -+CPU ?= $(shell uname -m) -+ -+# look up KDIR based om CPU selection -+KDIR ?= $(KDIR-$(CPU)) -+ -+ifeq ($(KDIR),) -+$(error No KDIR found for platform $(CPU)) -+endif -+ -+all: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) -+ -+kernelrelease: -+ $(MAKE) -C $(KDIR) kernelrelease -+ -+clean: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -+ -+endif -diff --git a/drivers/gpu/arm/mali400/umplock/umplock_driver.c b/drivers/gpu/arm/mali400/umplock/umplock_driver.c -new file mode 100755 -index 000000000..173f4d9bb ---- /dev/null -+++ b/drivers/gpu/arm/mali400/umplock/umplock_driver.c -@@ -0,0 +1,618 @@ -+/* -+ * Copyright (C) 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "umplock_ioctl.h" -+#include -+ -+#define MAX_ITEMS 1024 -+#define MAX_PIDS 128 -+ -+typedef struct lock_cmd_priv { -+ uint32_t msg[128]; /*ioctl args*/ -+ u32 pid; /*process id*/ -+} _lock_cmd_priv; -+ -+typedef struct lock_ref { -+ int ref_count; -+ u32 pid; -+ u32 down_count; -+} _lock_ref; -+ -+typedef struct umplock_item { -+ u32 secure_id; -+ u32 id_ref_count; -+ u32 owner; -+ _lock_access_usage usage; -+ _lock_ref references[MAX_PIDS]; -+ struct semaphore item_lock; -+} umplock_item; -+ -+typedef struct umplock_device_private { -+ struct mutex item_list_lock; -+ atomic_t sessions; -+ umplock_item items[MAX_ITEMS]; -+ u32 pids[MAX_PIDS]; -+} umplock_device_private; -+ -+struct umplock_device { -+ struct cdev cdev; -+ struct class *umplock_class; -+}; -+ -+static struct umplock_device umplock_device; -+static umplock_device_private device; -+static dev_t umplock_dev; -+static char umplock_dev_name[] = "umplock"; -+ -+int umplock_debug_level = 0; -+module_param(umplock_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */ -+MODULE_PARM_DESC(umplock_debug_level, "set umplock_debug_level to print debug messages"); -+ -+#define PDEBUG(level, fmt, args...) do { if ((level) <= umplock_debug_level) printk(KERN_DEBUG "umplock: " fmt, ##args); } while (0) -+#define PERROR(fmt, args...) do { printk(KERN_ERR "umplock: " fmt, ##args); } while (0) -+ -+int umplock_find_item(u32 secure_id) -+{ -+ int i; -+ for (i = 0; i < MAX_ITEMS; i++) { -+ if (device.items[i].secure_id == secure_id) { -+ return i; -+ } -+ } -+ -+ return -1; -+} -+ -+static int umplock_find_item_by_pid(_lock_cmd_priv *lock_cmd, int *item_slot, int *ref_slot) -+{ -+ _lock_item_s *lock_item; -+ int i, j; -+ -+ lock_item = (_lock_item_s *)&lock_cmd->msg; -+ -+ i = umplock_find_item(lock_item->secure_id); -+ -+ if (i < 0) { -+ return -1; -+ } -+ -+ for (j = 0; j < MAX_PIDS; j++) { -+ if (device.items[i].references[j].pid == lock_cmd->pid) { -+ *item_slot = i; -+ *ref_slot = j; -+ return 0; -+ } -+ } -+ return -1 ; -+} -+ -+static int umplock_find_client_valid(u32 pid) -+{ -+ int i; -+ -+ if (pid == 0) { -+ return -1; -+ } -+ -+ for (i = 0; i < MAX_PIDS; i++) { -+ if (device.pids[i] == pid) { -+ return i; -+ } -+ } -+ -+ return -1; -+} -+ -+static int do_umplock_create_locked(_lock_cmd_priv *lock_cmd) -+{ -+ int i_index, ref_index; -+ int ret; -+ _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; -+ -+ i_index = ref_index = -1; -+ -+ ret = umplock_find_client_valid(lock_cmd->pid); -+ if (ret < 0) { -+ /*lock request from an invalid client pid, do nothing*/ -+ return -EINVAL; -+ } -+ -+ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); -+ if (ret >= 0) { -+ } else if ((i_index = umplock_find_item(lock_item->secure_id)) >= 0) { -+ for (ref_index = 0; ref_index < MAX_PIDS; ref_index++) { -+ if (device.items[i_index].references[ref_index].pid == 0) { -+ break; -+ } -+ } -+ if (ref_index < MAX_PIDS) { -+ device.items[i_index].references[ref_index].pid = lock_cmd->pid; -+ device.items[i_index].references[ref_index].ref_count = 0; -+ device.items[i_index].references[ref_index].down_count = 0; -+ } else { -+ PERROR("whoops, item ran out of available reference slots\n"); -+ return -EINVAL; -+ -+ } -+ } else { -+ i_index = umplock_find_item(0); -+ -+ if (i_index >= 0) { -+ device.items[i_index].secure_id = lock_item->secure_id; -+ device.items[i_index].id_ref_count = 0; -+ device.items[i_index].usage = lock_item->usage; -+ device.items[i_index].references[0].pid = lock_cmd->pid; -+ device.items[i_index].references[0].ref_count = 0; -+ device.items[i_index].references[0].down_count = 0; -+ sema_init(&device.items[i_index].item_lock, 1); -+ } else { -+ PERROR("whoops, ran out of available slots\n"); -+ return -EINVAL; -+ } -+ } -+ -+ return 0; -+} -+/** IOCTLs **/ -+ -+static int do_umplock_create(_lock_cmd_priv *lock_cmd) -+{ -+ return 0; -+} -+ -+static int do_umplock_process(_lock_cmd_priv *lock_cmd) -+{ -+ int ret, i_index, ref_index; -+ _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; -+ -+ mutex_lock(&device.item_list_lock); -+ -+ if (0 == lock_item->secure_id) { -+ PERROR("IOCTL_UMPLOCK_PROCESS called with secure_id is 0, pid: %d\n", lock_cmd->pid); -+ mutex_unlock(&device.item_list_lock); -+ return -EINVAL; -+ } -+ -+ ret = do_umplock_create_locked(lock_cmd); -+ if (ret < 0) { -+ mutex_unlock(&device.item_list_lock); -+ return -EINVAL; -+ } -+ -+ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); -+ if (ret < 0) { -+ /*fail to find a item*/ -+ PERROR("IOCTL_UMPLOCK_PROCESS called with invalid parameter, pid: %d\n", lock_cmd->pid); -+ mutex_unlock(&device.item_list_lock); -+ return -EINVAL; -+ } -+ device.items[i_index].references[ref_index].ref_count++; -+ device.items[i_index].id_ref_count++; -+ PDEBUG(1, "try to lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); -+ -+ if (lock_cmd->pid == device.items[i_index].owner) { -+ PDEBUG(1, "already own the lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); -+ mutex_unlock(&device.item_list_lock); -+ return 0; -+ } -+ -+ device.items[i_index].references[ref_index].down_count++; -+ mutex_unlock(&device.item_list_lock); -+ if (down_interruptible(&device.items[i_index].item_lock)) { -+ /*wait up without hold the umplock. restore previous state and return*/ -+ mutex_lock(&device.item_list_lock); -+ device.items[i_index].references[ref_index].ref_count--; -+ device.items[i_index].id_ref_count--; -+ device.items[i_index].references[ref_index].down_count--; -+ if (0 == device.items[i_index].references[ref_index].ref_count) { -+ device.items[i_index].references[ref_index].pid = 0; -+ if (0 == device.items[i_index].id_ref_count) { -+ PDEBUG(1, "release item, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); -+ device.items[i_index].secure_id = 0; -+ } -+ } -+ -+ PERROR("failed lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); -+ -+ mutex_unlock(&device.item_list_lock); -+ return -ERESTARTSYS; -+ } -+ -+ mutex_lock(&device.item_list_lock); -+ PDEBUG(1, "got lock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); -+ device.items[i_index].owner = lock_cmd->pid; -+ mutex_unlock(&device.item_list_lock); -+ -+ return 0; -+} -+ -+static int do_umplock_release(_lock_cmd_priv *lock_cmd) -+{ -+ int ret, i_index, ref_index, call_up; -+ _lock_item_s *lock_item = (_lock_item_s *)&lock_cmd->msg; -+ -+ mutex_lock(&device.item_list_lock); -+ -+ if (0 == lock_item->secure_id) { -+ PERROR("IOCTL_UMPLOCK_RELEASE called with secure_id is 0, pid: %d\n", lock_cmd->pid); -+ mutex_unlock(&device.item_list_lock); -+ return -EINVAL; -+ } -+ -+ ret = umplock_find_client_valid(lock_cmd->pid); -+ if (ret < 0) { -+ /*lock request from an invalid client pid, do nothing*/ -+ mutex_unlock(&device.item_list_lock); -+ return -EPERM; -+ } -+ -+ i_index = ref_index = -1; -+ -+ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); -+ if (ret < 0) { -+ /*fail to find item*/ -+ PERROR("IOCTL_UMPLOCK_RELEASE called with invalid parameter pid: %d, secid: 0x%x\n", lock_cmd->pid, lock_item->secure_id); -+ mutex_unlock(&device.item_list_lock); -+ return -EINVAL; -+ } -+ -+ /* if the lock is not owned by this process */ -+ if (lock_cmd->pid != device.items[i_index].owner) { -+ mutex_unlock(&device.item_list_lock); -+ return -EPERM; -+ } -+ -+ /* if the ref_count is 0, that means nothing to unlock, just return */ -+ if (0 == device.items[i_index].references[ref_index].ref_count) { -+ mutex_unlock(&device.item_list_lock); -+ return 0; -+ } -+ -+ device.items[i_index].references[ref_index].ref_count--; -+ device.items[i_index].id_ref_count--; -+ PDEBUG(1, "unlock, pid: %d, secure_id: 0x%x, ref_count: %d\n", lock_cmd->pid, lock_item->secure_id, device.items[i_index].references[ref_index].ref_count); -+ -+ call_up = 0; -+ if (device.items[i_index].references[ref_index].down_count > 1) { -+ call_up = 1; -+ device.items[i_index].references[ref_index].down_count--; -+ } -+ if (0 == device.items[i_index].references[ref_index].ref_count) { -+ device.items[i_index].references[ref_index].pid = 0; -+ if (0 == device.items[i_index].id_ref_count) { -+ PDEBUG(1, "release item, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); -+ device.items[i_index].secure_id = 0; -+ } -+ device.items[i_index].owner = 0; -+ call_up = 1; -+ } -+ if (call_up) { -+ PDEBUG(1, "call up, pid: %d, secure_id: 0x%x\n", lock_cmd->pid, lock_item->secure_id); -+ up(&device.items[i_index].item_lock); -+ } -+ mutex_unlock(&device.item_list_lock); -+ -+ return 0; -+} -+ -+static int do_umplock_zap(void) -+{ -+ int i; -+ -+ PDEBUG(1, "ZAP ALL ENTRIES!\n"); -+ -+ mutex_lock(&device.item_list_lock); -+ -+ for (i = 0; i < MAX_ITEMS; i++) { -+ device.items[i].secure_id = 0; -+ memset(&device.items[i].references, 0, sizeof(_lock_ref) * MAX_PIDS); -+ sema_init(&device.items[i].item_lock, 1); -+ } -+ -+ for (i = 0; i < MAX_PIDS; i++) { -+ device.pids[i] = 0; -+ } -+ mutex_unlock(&device.item_list_lock); -+ -+ return 0; -+} -+ -+static int do_umplock_dump(void) -+{ -+ int i, j; -+ -+ mutex_lock(&device.item_list_lock); -+ PERROR("dump all the items begin\n"); -+ for (i = 0; i < MAX_ITEMS; i++) { -+ for (j = 0; j < MAX_PIDS; j++) { -+ if (device.items[i].secure_id != 0 && device.items[i].references[j].pid != 0) { -+ PERROR("item[%d]->secure_id=0x%x, owner=%d\t reference[%d].ref_count=%d.pid=%d\n", -+ i, -+ device.items[i].secure_id, -+ device.items[i].owner, -+ j, -+ device.items[i].references[j].ref_count, -+ device.items[i].references[j].pid); -+ } -+ } -+ } -+ PERROR("dump all the items end\n"); -+ mutex_unlock(&device.item_list_lock); -+ -+ return 0; -+} -+ -+int do_umplock_client_add(_lock_cmd_priv *lock_cmd) -+{ -+ int i; -+ mutex_lock(&device.item_list_lock); -+ for (i = 0; i < MAX_PIDS; i++) { -+ if (device.pids[i] == lock_cmd->pid) { -+ mutex_unlock(&device.item_list_lock); -+ return 0; -+ } -+ } -+ for (i = 0; i < MAX_PIDS; i++) { -+ if (device.pids[i] == 0) { -+ device.pids[i] = lock_cmd->pid; -+ break; -+ } -+ } -+ mutex_unlock(&device.item_list_lock); -+ if (i == MAX_PIDS) { -+ PERROR("Oops, Run out of client slots\n "); -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+int do_umplock_client_delete(_lock_cmd_priv *lock_cmd) -+{ -+ int p_index = -1, i_index = -1, ref_index = -1; -+ int ret; -+ _lock_item_s *lock_item; -+ lock_item = (_lock_item_s *)&lock_cmd->msg; -+ -+ mutex_lock(&device.item_list_lock); -+ p_index = umplock_find_client_valid(lock_cmd->pid); -+ /*lock item pid is not valid.*/ -+ if (p_index < 0) { -+ mutex_unlock(&device.item_list_lock); -+ return 0; -+ } -+ -+ /*walk through umplock item list and release reference attached to this client*/ -+ for (i_index = 0; i_index < MAX_ITEMS; i_index++) { -+ lock_item->secure_id = device.items[i_index].secure_id; -+ -+ /*find the item index and reference slot for the lock_item*/ -+ ret = umplock_find_item_by_pid(lock_cmd, &i_index, &ref_index); -+ -+ if (ret < 0) { -+ /*client has no reference on this umplock item, skip*/ -+ continue; -+ } -+ while (device.items[i_index].references[ref_index].ref_count) { -+ /*release references on this client*/ -+ -+ PDEBUG(1, "delete client, pid: %d, ref_count: %d\n", lock_cmd->pid, device.items[i_index].references[ref_index].ref_count); -+ -+ mutex_unlock(&device.item_list_lock); -+ do_umplock_release(lock_cmd); -+ mutex_lock(&device.item_list_lock); -+ } -+ } -+ -+ /*remove the pid from umplock valid pid list*/ -+ device.pids[p_index] = 0; -+ mutex_unlock(&device.item_list_lock); -+ -+ return 0; -+} -+ -+static long umplock_driver_ioctl(struct file *f, unsigned int cmd, unsigned long arg) -+{ -+ int ret; -+ uint32_t size = _IOC_SIZE(cmd); -+ _lock_cmd_priv lock_cmd ; -+ -+ if (_IOC_TYPE(cmd) != LOCK_IOCTL_GROUP) { -+ return -ENOTTY; -+ } -+ -+ if (_IOC_NR(cmd) >= LOCK_IOCTL_MAX_CMDS) { -+ return -ENOTTY; -+ } -+ -+ switch (cmd) { -+ case LOCK_IOCTL_CREATE: -+ if (size != sizeof(_lock_item_s)) { -+ return -ENOTTY; -+ } -+ -+ if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { -+ return -EFAULT; -+ } -+ lock_cmd.pid = (u32)current->tgid; -+ ret = do_umplock_create(&lock_cmd); -+ if (ret) { -+ return ret; -+ } -+ return 0; -+ -+ case LOCK_IOCTL_PROCESS: -+ if (size != sizeof(_lock_item_s)) { -+ return -ENOTTY; -+ } -+ -+ if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { -+ return -EFAULT; -+ } -+ lock_cmd.pid = (u32)current->tgid; -+ return do_umplock_process(&lock_cmd); -+ -+ case LOCK_IOCTL_RELEASE: -+ if (size != sizeof(_lock_item_s)) { -+ return -ENOTTY; -+ } -+ -+ if (copy_from_user(&lock_cmd.msg, (void __user *)arg, size)) { -+ return -EFAULT; -+ } -+ lock_cmd.pid = (u32)current->tgid; -+ ret = do_umplock_release(&lock_cmd); -+ if (ret) { -+ return ret; -+ } -+ return 0; -+ -+ case LOCK_IOCTL_ZAP: -+ do_umplock_zap(); -+ return 0; -+ -+ case LOCK_IOCTL_DUMP: -+ do_umplock_dump(); -+ return 0; -+ } -+ -+ return -ENOIOCTLCMD; -+} -+ -+static int umplock_driver_open(struct inode *inode, struct file *filp) -+{ -+ _lock_cmd_priv lock_cmd; -+ -+ atomic_inc(&device.sessions); -+ PDEBUG(1, "OPEN SESSION (%i references)\n", atomic_read(&device.sessions)); -+ -+ lock_cmd.pid = (u32)current->tgid; -+ do_umplock_client_add(&lock_cmd); -+ -+ return 0; -+} -+ -+static int umplock_driver_release(struct inode *inode, struct file *filp) -+{ -+ int sessions = 0; -+ _lock_cmd_priv lock_cmd; -+ -+ lock_cmd.pid = (u32)current->tgid; -+ do_umplock_client_delete(&lock_cmd); -+ -+ mutex_lock(&device.item_list_lock); -+ atomic_dec(&device.sessions); -+ sessions = atomic_read(&device.sessions); -+ PDEBUG(1, "CLOSE SESSION (%i references)\n", sessions); -+ mutex_unlock(&device.item_list_lock); -+ if (sessions == 0) { -+ do_umplock_zap(); -+ } -+ -+ return 0; -+} -+ -+static struct file_operations umplock_fops = { -+ .owner = THIS_MODULE, -+ .open = umplock_driver_open, -+ .release = umplock_driver_release, -+ .unlocked_ioctl = umplock_driver_ioctl, -+}; -+ -+int umplock_device_initialize(void) -+{ -+ int err; -+ -+ err = alloc_chrdev_region(&umplock_dev, 0, 1, umplock_dev_name); -+ -+ if (0 == err) { -+ memset(&umplock_device, 0, sizeof(umplock_device)); -+ cdev_init(&umplock_device.cdev, &umplock_fops); -+ umplock_device.cdev.owner = THIS_MODULE; -+ umplock_device.cdev.ops = &umplock_fops; -+ -+ err = cdev_add(&umplock_device.cdev, umplock_dev, 1); -+ if (0 == err) { -+ umplock_device.umplock_class = class_create(THIS_MODULE, umplock_dev_name); -+ if (IS_ERR(umplock_device.umplock_class)) { -+ err = PTR_ERR(umplock_device.umplock_class); -+ } else { -+ struct device *mdev; -+ mdev = device_create(umplock_device.umplock_class, NULL, umplock_dev, NULL, umplock_dev_name); -+ if (!IS_ERR(mdev)) { -+ return 0; /* all ok */ -+ } -+ -+ err = PTR_ERR(mdev); -+ class_destroy(umplock_device.umplock_class); -+ } -+ cdev_del(&umplock_device.cdev); -+ } -+ -+ unregister_chrdev_region(umplock_dev, 1); -+ } else { -+ PERROR("alloc chardev region failed\n"); -+ } -+ -+ return err; -+} -+ -+void umplock_device_terminate(void) -+{ -+ device_destroy(umplock_device.umplock_class, umplock_dev); -+ class_destroy(umplock_device.umplock_class); -+ -+ cdev_del(&umplock_device.cdev); -+ unregister_chrdev_region(umplock_dev, 1); -+} -+ -+static int __init umplock_initialize_module(void) -+{ -+ PDEBUG(1, "Inserting UMP lock device driver. Compiled: %s, time: %s\n", __DATE__, __TIME__); -+ -+ mutex_init(&device.item_list_lock); -+ if (umplock_device_initialize() != 0) { -+ PERROR("UMP lock device driver init failed\n"); -+ return -ENOTTY; -+ } -+ memset(&device.items, 0, sizeof(umplock_item) * MAX_ITEMS); -+ memset(&device.pids, 0, sizeof(u32) * MAX_PIDS); -+ atomic_set(&device.sessions, 0); -+ -+ PDEBUG(1, "UMP lock device driver loaded\n"); -+ -+ return 0; -+} -+ -+static void __exit umplock_cleanup_module(void) -+{ -+ PDEBUG(1, "unloading UMP lock module\n"); -+ -+ memset(&device.items, 0, sizeof(umplock_item) * MAX_ITEMS); -+ memset(&device.pids, 0, sizeof(u32) * MAX_PIDS); -+ umplock_device_terminate(); -+ mutex_destroy(&device.item_list_lock); -+ -+ PDEBUG(1, "UMP lock module unloaded\n"); -+} -+ -+module_init(umplock_initialize_module); -+module_exit(umplock_cleanup_module); -+ -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("ARM Ltd."); -+MODULE_DESCRIPTION("ARM UMP locker"); -diff --git a/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h -new file mode 100755 -index 000000000..8afdaad70 ---- /dev/null -+++ b/drivers/gpu/arm/mali400/umplock/umplock_ioctl.h -@@ -0,0 +1,66 @@ -+/* -+ * Copyright (C) 2012-2013, 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the GNU General Public License version 2 -+ * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained from Free Software -+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -+ */ -+ -+#ifndef __UMPLOCK_IOCTL_H__ -+#define __UMPLOCK_IOCTL_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include -+#include -+ -+#ifndef __user -+#define __user -+#endif -+ -+ -+/** -+ * @file umplock_ioctl.h -+ * This file describes the interface needed to use the Linux device driver. -+ * The interface is used by the userpace Mali DDK. -+ */ -+ -+typedef enum { -+ _LOCK_ACCESS_RENDERABLE = 1, -+ _LOCK_ACCESS_TEXTURE, -+ _LOCK_ACCESS_CPU_WRITE, -+ _LOCK_ACCESS_CPU_READ, -+} _lock_access_usage; -+ -+typedef struct _lock_item_s { -+ unsigned int secure_id; -+ _lock_access_usage usage; -+} _lock_item_s; -+ -+ -+#define LOCK_IOCTL_GROUP 0x91 -+ -+#define _LOCK_IOCTL_CREATE_CMD 0 /* create kernel lock item */ -+#define _LOCK_IOCTL_PROCESS_CMD 1 /* process kernel lock item */ -+#define _LOCK_IOCTL_RELEASE_CMD 2 /* release kernel lock item */ -+#define _LOCK_IOCTL_ZAP_CMD 3 /* clean up all kernel lock items */ -+#define _LOCK_IOCTL_DUMP_CMD 4 /* dump all the items */ -+ -+#define LOCK_IOCTL_MAX_CMDS 5 -+ -+#define LOCK_IOCTL_CREATE _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_CREATE_CMD, _lock_item_s ) -+#define LOCK_IOCTL_PROCESS _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_PROCESS_CMD, _lock_item_s ) -+#define LOCK_IOCTL_RELEASE _IOW( LOCK_IOCTL_GROUP, _LOCK_IOCTL_RELEASE_CMD, _lock_item_s ) -+#define LOCK_IOCTL_ZAP _IO ( LOCK_IOCTL_GROUP, _LOCK_IOCTL_ZAP_CMD ) -+#define LOCK_IOCTL_DUMP _IO ( LOCK_IOCTL_GROUP, _LOCK_IOCTL_DUMP_CMD ) -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __UMPLOCK_IOCTL_H__ */ -+ -diff --git a/drivers/gpu/arm/midgard/Kbuild b/drivers/gpu/arm/midgard/Kbuild -new file mode 100755 -index 000000000..b2c2bbcda ---- /dev/null -+++ b/drivers/gpu/arm/midgard/Kbuild -@@ -0,0 +1,221 @@ -+# -+# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+KBUILD_CFLAGS += -include rename.h -+ -+# Driver version string which is returned to userspace via an ioctl -+MALI_RELEASE_NAME ?= "r18p0-01rel0" -+ -+# Paths required for build -+ -+# make $(src) as absolute path if it isn't already, by prefixing $(srctree) -+src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) -+KBASE_PATH = $(src) -+KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy -+UMP_PATH = $(src)/../../../base -+ -+ifeq ($(CONFIG_MALI_ERROR_INJECTION),y) -+MALI_ERROR_INJECT_ON = 1 -+endif -+ -+# Set up defaults if not defined by build system -+MALI_CUSTOMER_RELEASE ?= 1 -+MALI_UNIT_TEST ?= 0 -+MALI_KERNEL_TEST_API ?= 0 -+MALI_ERROR_INJECT_ON ?= 0 -+MALI_MOCK_TEST ?= 0 -+MALI_COVERAGE ?= 0 -+MALI_INSTRUMENTATION_LEVEL ?= 0 -+# This workaround is for what seems to be a compiler bug we observed in -+# GCC 4.7 on AOSP 4.3. The bug caused an intermittent failure compiling -+# the "_Pragma" syntax, where an error message is returned: -+# -+# "internal compiler error: unspellable token PRAGMA" -+# -+# This regression has thus far only been seen on the GCC 4.7 compiler bundled -+# with AOSP 4.3.0. So this makefile, intended for in-tree kernel builds -+# which are not known to be used with AOSP, is hardcoded to disable the -+# workaround, i.e. set the define to 0. -+MALI_GCC_WORKAROUND_MIDCOM_4598 ?= 0 -+ -+# Set up our defines, which will be passed to gcc -+DEFINES = \ -+ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ -+ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ -+ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -+ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ -+ -DMALI_MOCK_TEST=$(MALI_MOCK_TEST) \ -+ -DMALI_COVERAGE=$(MALI_COVERAGE) \ -+ -DMALI_INSTRUMENTATION_LEVEL=$(MALI_INSTRUMENTATION_LEVEL) \ -+ -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ -+ -DMALI_GCC_WORKAROUND_MIDCOM_4598=$(MALI_GCC_WORKAROUND_MIDCOM_4598) -+ -+ifeq ($(KBUILD_EXTMOD),) -+# in-tree -+DEFINES +=-DMALI_KBASE_THIRDPARTY_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME) -+else -+# out-of-tree -+DEFINES +=-DMALI_KBASE_THIRDPARTY_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME) -+endif -+ -+DEFINES += -I$(srctree)/drivers/staging/android -+ -+# Use our defines when compiling -+ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux -+subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(OSK_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux -+ -+SRC := \ -+ mali_kbase_device.c \ -+ mali_kbase_cache_policy.c \ -+ mali_kbase_mem.c \ -+ mali_kbase_mmu.c \ -+ mali_kbase_ctx_sched.c \ -+ mali_kbase_jd.c \ -+ mali_kbase_jd_debugfs.c \ -+ mali_kbase_jm.c \ -+ mali_kbase_gpuprops.c \ -+ mali_kbase_js.c \ -+ mali_kbase_js_ctx_attr.c \ -+ mali_kbase_event.c \ -+ mali_kbase_context.c \ -+ mali_kbase_pm.c \ -+ mali_kbase_config.c \ -+ mali_kbase_vinstr.c \ -+ mali_kbase_softjobs.c \ -+ mali_kbase_10969_workaround.c \ -+ mali_kbase_hw.c \ -+ mali_kbase_utility.c \ -+ mali_kbase_debug.c \ -+ mali_kbase_trace_timeline.c \ -+ mali_kbase_gpu_memory_debugfs.c \ -+ mali_kbase_mem_linux.c \ -+ mali_kbase_core_linux.c \ -+ mali_kbase_replay.c \ -+ mali_kbase_mem_profile_debugfs.c \ -+ mali_kbase_mmu_mode_lpae.c \ -+ mali_kbase_mmu_mode_aarch64.c \ -+ mali_kbase_disjoint_events.c \ -+ mali_kbase_gator_api.c \ -+ mali_kbase_debug_mem_view.c \ -+ mali_kbase_debug_job_fault.c \ -+ mali_kbase_smc.c \ -+ mali_kbase_mem_pool.c \ -+ mali_kbase_mem_pool_debugfs.c \ -+ mali_kbase_tlstream.c \ -+ mali_kbase_strings.c \ -+ mali_kbase_as_fault_debugfs.c \ -+ mali_kbase_regs_history_debugfs.c -+ -+ -+ -+ -+ifeq ($(MALI_UNIT_TEST),1) -+ SRC += mali_kbase_tlstream_test.c -+endif -+ -+ifeq ($(MALI_CUSTOMER_RELEASE),0) -+ SRC += mali_kbase_regs_dump_debugfs.c -+endif -+ -+ -+ccflags-y += -I$(KBASE_PATH) -+ -+ifeq ($(CONFIG_MALI_PLATFORM_FAKE),y) -+ SRC += mali_kbase_platform_fake.c -+ -+ ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS),y) -+ SRC += platform/vexpress/mali_kbase_config_vexpress.c \ -+ platform/vexpress/mali_kbase_cpu_vexpress.c -+ ccflags-y += -I$(src)/platform/vexpress -+ endif -+ -+ ifeq ($(CONFIG_MALI_PLATFORM_RTSM_VE),y) -+ SRC += platform/rtsm_ve/mali_kbase_config_vexpress.c -+ ccflags-y += -I$(src)/platform/rtsm_ve -+ endif -+ -+ ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_1XV7_A57),y) -+ SRC += platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c -+ ccflags-y += -I$(src)/platform/vexpress_1xv7_a57 -+ endif -+ -+ ifeq ($(CONFIG_MALI_PLATFORM_VEXPRESS_6XVIRTEX7_10MHZ),y) -+ SRC += platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c \ -+ platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c -+ ccflags-y += -I$(src)/platform/vexpress_6xvirtex7_10mhz -+ endif -+endif # CONFIG_MALI_PLATFORM_FAKE=y -+ -+# Tell the Linux build system from which .o file to create the kernel module -+obj-$(CONFIG_MALI_MIDGARD) += midgard_kbase.o -+ -+# Tell the Linux build system to enable building of our .c files -+midgard_kbase-y := $(SRC:.c=.o) -+ -+ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY),y) -+ # Kconfig passes in the name with quotes for in-tree builds - remove them. -+ platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_THIRDPARTY_NAME)) -+ MALI_PLATFORM_THIRDPARTY_DIR := platform/$(platform_name) -+ ccflags-y += -I$(src)/$(MALI_PLATFORM_THIRDPARTY_DIR) -+ include $(src)/$(MALI_PLATFORM_THIRDPARTY_DIR)/Kbuild -+endif -+ -+ifeq ($(CONFIG_MALI_DEVFREQ),y) -+ ifeq ($(CONFIG_DEVFREQ_THERMAL),y) -+ include $(src)/ipa/Kbuild -+ endif -+endif -+ -+midgard_kbase-$(CONFIG_MALI_DMA_FENCE) += \ -+ mali_kbase_dma_fence.o \ -+ mali_kbase_fence.o -+midgard_kbase-$(CONFIG_SYNC) += \ -+ mali_kbase_sync_android.o \ -+ mali_kbase_sync_common.o -+midgard_kbase-$(CONFIG_SYNC_FILE) += \ -+ mali_kbase_sync_file.o \ -+ mali_kbase_sync_common.o \ -+ mali_kbase_fence.o -+ -+MALI_BACKEND_PATH ?= backend -+CONFIG_MALI_BACKEND ?= gpu -+CONFIG_MALI_BACKEND_REAL ?= $(CONFIG_MALI_BACKEND) -+ -+ifeq ($(MALI_MOCK_TEST),1) -+ifeq ($(CONFIG_MALI_BACKEND_REAL),gpu) -+# Test functionality -+midgard_kbase-y += tests/internal/src/mock/mali_kbase_pm_driver_mock.o -+endif -+endif -+ -+include $(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL)/Kbuild -+midgard_kbase-y += $(BACKEND:.c=.o) -+ -+ -+ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL) -+subdir-ccflags-y += -I$(src)/$(MALI_BACKEND_PATH)/$(CONFIG_MALI_BACKEND_REAL) -+ -+# Default to devicetree platform if neither a fake platform or a thirdparty -+# platform is configured. -+ifeq ($(CONFIG_MALI_PLATFORM_THIRDPARTY)$(CONFIG_MALI_PLATFORM_FAKE),) -+CONFIG_MALI_PLATFORM_DEVICETREE := y -+endif -+ -+midgard_kbase-$(CONFIG_MALI_PLATFORM_DEVICETREE) += \ -+ platform/devicetree/mali_kbase_runtime_pm.o \ -+ platform/devicetree/mali_kbase_config_devicetree.o -+ccflags-$(CONFIG_MALI_PLATFORM_DEVICETREE) += -I$(src)/platform/devicetree -+ -+# For kutf and mali_kutf_irq_latency_test -+obj-$(CONFIG_MALI_KUTF) += tests/ -diff --git a/drivers/gpu/arm/midgard/Kconfig b/drivers/gpu/arm/midgard/Kconfig -new file mode 100755 -index 000000000..1b28bb73a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/Kconfig -@@ -0,0 +1,248 @@ -+# -+# (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ -+menuconfig MALI_MIDGARD -+ tristate "Mali Midgard series support" -+ select GPU_TRACEPOINTS if ANDROID -+ default n -+ help -+ Enable this option to build support for a ARM Mali Midgard GPU. -+ -+ To compile this driver as a module, choose M here: -+ this will generate a single module, called mali_kbase. -+ -+config MALI_GATOR_SUPPORT -+ bool "Streamline support via Gator" -+ depends on MALI_MIDGARD -+ default n -+ help -+ Adds diagnostic support for use with the ARM Streamline Performance Analyzer. -+ You will need the Gator device driver already loaded before loading this driver when enabling -+ Streamline debug support. -+ This is a legacy interface required by older versions of Streamline. -+ -+config MALI_MIDGARD_DVFS -+ bool "Enable legacy DVFS" -+ depends on MALI_MIDGARD && !MALI_DEVFREQ && !MALI_PLATFORM_DEVICETREE -+ default n -+ help -+ Choose this option to enable legacy DVFS in the Mali Midgard DDK. -+ -+config MALI_MIDGARD_ENABLE_TRACE -+ bool "Enable kbase tracing" -+ depends on MALI_MIDGARD -+ default n -+ help -+ Enables tracing in kbase. Trace log available through -+ the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled -+ -+config MALI_DEVFREQ -+ bool "devfreq support for Mali" -+ depends on MALI_MIDGARD && PM_DEVFREQ -+ help -+ Support devfreq for Mali. -+ -+ Using the devfreq framework and, by default, the simpleondemand -+ governor, the frequency of Mali will be dynamically selected from the -+ available OPPs. -+ -+config MALI_DMA_FENCE -+ bool "DMA_BUF fence support for Mali" -+ depends on MALI_MIDGARD && !KDS -+ default n -+ help -+ Support DMA_BUF fences for Mali. -+ -+ This option should only be enabled if KDS is not present and -+ the Linux Kernel has built in support for DMA_BUF fences. -+ -+# MALI_EXPERT configuration options -+ -+menuconfig MALI_EXPERT -+ depends on MALI_MIDGARD -+ bool "Enable Expert Settings" -+ default n -+ help -+ Enabling this option and modifying the default settings may produce a driver with performance or -+ other limitations. -+ -+config MALI_CORESTACK -+ bool "Support controlling power to the GPU core stack" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Enabling this feature on supported GPUs will let the driver powering -+ on/off the GPU core stack independently without involving the Power -+ Domain Controller. This should only be enabled on platforms which -+ integration of the PDC to the Mali GPU is known to be problematic. -+ This feature is currently only supported on t-Six and t-HEx GPUs. -+ -+ If unsure, say N. -+ -+config MALI_PRFCNT_SET_SECONDARY -+ bool "Use secondary set of performance counters" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Select this option to use secondary set of performance counters. Kernel -+ features that depend on an access to the primary set of counters may -+ become unavailable. Enabling this option will prevent power management -+ from working optimally and may cause instrumentation tools to return -+ bogus results. -+ -+ If unsure, say N. -+ -+config MALI_PLATFORM_FAKE -+ bool "Enable fake platform device support" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ When you start to work with the Mali Midgard series device driver the platform-specific code of -+ the Linux kernel for your platform may not be complete. In this situation the kernel device driver -+ supports creating the platform device outside of the Linux platform-specific code. -+ Enable this option if would like to use a platform device configuration from within the device driver. -+ -+choice -+ prompt "Platform configuration" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default MALI_PLATFORM_DEVICETREE -+ help -+ Select the SOC platform that contains a Mali Midgard GPU -+ -+config MALI_PLATFORM_DEVICETREE -+ bool "Device Tree platform" -+ depends on OF -+ help -+ Select this option to use Device Tree with the Mali driver. -+ -+ When using this option the Mali driver will get the details of the -+ GPU hardware from the Device Tree. This means that the same driver -+ binary can run on multiple platforms as long as all the GPU hardware -+ details are described in the device tree. -+ -+ Device Tree is the recommended method for the Mali driver platform -+ integration. -+ -+config MALI_PLATFORM_VEXPRESS -+ depends on ARCH_VEXPRESS && (ARCH_VEXPRESS_CA9X4 || ARCH_VEXPRESS_CA15X4) -+ bool "Versatile Express" -+config MALI_PLATFORM_VEXPRESS_VIRTEX7_40MHZ -+ depends on ARCH_VEXPRESS && (ARCH_VEXPRESS_CA9X4 || ARCH_VEXPRESS_CA15X4) -+ bool "Versatile Express w/Virtex7 @ 40Mhz" -+config MALI_PLATFORM_GOLDFISH -+ depends on ARCH_GOLDFISH -+ bool "Android Goldfish virtual CPU" -+config MALI_PLATFORM_PBX -+ depends on ARCH_REALVIEW && REALVIEW_EB_A9MP && MACH_REALVIEW_PBX -+ bool "Realview PBX-A9" -+config MALI_PLATFORM_THIRDPARTY -+ bool "Third Party Platform" -+endchoice -+ -+config MALI_PLATFORM_THIRDPARTY_NAME -+ depends on MALI_MIDGARD && MALI_PLATFORM_THIRDPARTY && MALI_EXPERT -+ string "Third party platform name" -+ help -+ Enter the name of a third party platform that is supported. The third part configuration -+ file must be in midgard/config/tpip/mali_kbase_config_xxx.c where xxx is the name -+ specified here. -+ -+config MALI_DEBUG -+ bool "Debug build" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Select this option for increased checking and reporting of errors. -+ -+config MALI_FENCE_DEBUG -+ bool "Debug sync fence usage" -+ depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE) -+ default y if MALI_DEBUG -+ help -+ Select this option to enable additional checking and reporting on the -+ use of sync fences in the Mali driver. -+ -+ This will add a 3s timeout to all sync fence waits in the Mali -+ driver, so that when work for Mali has been waiting on a sync fence -+ for a long time a debug message will be printed, detailing what fence -+ is causing the block, and which dependent Mali atoms are blocked as a -+ result of this. -+ -+ The timeout can be changed at runtime through the js_soft_timeout -+ device attribute, where the timeout is specified in milliseconds. -+ -+config MALI_NO_MALI -+ bool "No Mali" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ This can be used to test the driver in a simulated environment -+ whereby the hardware is not physically present. If the hardware is physically -+ present it will not be used. This can be used to test the majority of the -+ driver without needing actual hardware or for software benchmarking. -+ All calls to the simulated hardware will complete immediately as if the hardware -+ completed the task. -+ -+config MALI_ERROR_INJECT -+ bool "Error injection" -+ depends on MALI_MIDGARD && MALI_EXPERT && MALI_NO_MALI -+ default n -+ help -+ Enables insertion of errors to test module failure and recovery mechanisms. -+ -+config MALI_TRACE_TIMELINE -+ bool "Timeline tracing" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Enables timeline tracing through the kernel tracepoint system. -+ -+config MALI_SYSTEM_TRACE -+ bool "Enable system event tracing support" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Choose this option to enable system trace events for each -+ kbase event. This is typically used for debugging but has -+ minimal overhead when not in use. Enable only if you know what -+ you are doing. -+ -+config MALI_GPU_MMU_AARCH64 -+ bool "Use AArch64 page tables" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Use AArch64 format page tables for the GPU instead of LPAE-style. -+ The two formats have the same functionality and performance but a -+ future GPU may deprecate or remove the legacy LPAE-style format. -+ -+ The LPAE-style format is supported on all Midgard and current Bifrost -+ GPUs. Enabling AArch64 format restricts the driver to only supporting -+ Bifrost GPUs. -+ -+ If in doubt, say N. -+ -+config MALI_PWRSOFT_765 -+ bool "PWRSOFT-765 ticket" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ PWRSOFT-765 fixes devfreq cooling devices issues. However, they are -+ not merged in mainline kernel yet. So this define helps to guard those -+ parts of the code. -+ -+source "drivers/gpu/arm/midgard/platform/Kconfig" -+source "drivers/gpu/arm/midgard/tests/Kconfig" -diff --git a/drivers/gpu/arm/midgard/Makefile b/drivers/gpu/arm/midgard/Makefile -new file mode 100755 -index 000000000..9aa242c4f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/Makefile -@@ -0,0 +1,42 @@ -+# -+# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ -+KDIR ?= /lib/modules/$(shell uname -r)/build -+ -+BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. -+UMP_PATH_RELATIVE = $(CURDIR)/../../../base/ump -+KBASE_PATH_RELATIVE = $(CURDIR) -+KDS_PATH_RELATIVE = $(CURDIR)/../../../.. -+EXTRA_SYMBOLS = $(UMP_PATH_RELATIVE)/src/Module.symvers -+ -+ifeq ($(MALI_UNIT_TEST), 1) -+ EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers -+endif -+ -+ifeq ($(MALI_BUS_LOG), 1) -+#Add bus logger symbols -+EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers -+endif -+ -+# GPL driver supports KDS -+EXTRA_SYMBOLS += $(KDS_PATH_RELATIVE)/drivers/base/kds/Module.symvers -+ -+# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions -+all: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules -+ -+clean: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) clean -diff --git a/drivers/gpu/arm/midgard/Makefile.kbase b/drivers/gpu/arm/midgard/Makefile.kbase -new file mode 100755 -index 000000000..2bef9c25e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/Makefile.kbase -@@ -0,0 +1,17 @@ -+# -+# (C) COPYRIGHT 2010 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(OSK_PATH)/src/linux/include -I$(KBASE_PATH)/platform_$(PLATFORM) -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/Kbuild b/drivers/gpu/arm/midgard/backend/gpu/Kbuild -new file mode 100755 -index 000000000..5f700e9b6 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/Kbuild -@@ -0,0 +1,60 @@ -+# -+# (C) COPYRIGHT 2014,2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+BACKEND += \ -+ backend/gpu/mali_kbase_cache_policy_backend.c \ -+ backend/gpu/mali_kbase_device_hw.c \ -+ backend/gpu/mali_kbase_gpu.c \ -+ backend/gpu/mali_kbase_gpuprops_backend.c \ -+ backend/gpu/mali_kbase_debug_job_fault_backend.c \ -+ backend/gpu/mali_kbase_irq_linux.c \ -+ backend/gpu/mali_kbase_instr_backend.c \ -+ backend/gpu/mali_kbase_jm_as.c \ -+ backend/gpu/mali_kbase_jm_hw.c \ -+ backend/gpu/mali_kbase_jm_rb.c \ -+ backend/gpu/mali_kbase_js_affinity.c \ -+ backend/gpu/mali_kbase_js_backend.c \ -+ backend/gpu/mali_kbase_mmu_hw_direct.c \ -+ backend/gpu/mali_kbase_pm_backend.c \ -+ backend/gpu/mali_kbase_pm_driver.c \ -+ backend/gpu/mali_kbase_pm_metrics.c \ -+ backend/gpu/mali_kbase_pm_ca.c \ -+ backend/gpu/mali_kbase_pm_ca_fixed.c \ -+ backend/gpu/mali_kbase_pm_always_on.c \ -+ backend/gpu/mali_kbase_pm_coarse_demand.c \ -+ backend/gpu/mali_kbase_pm_demand.c \ -+ backend/gpu/mali_kbase_pm_policy.c \ -+ backend/gpu/mali_kbase_time.c -+ -+ifeq ($(MALI_CUSTOMER_RELEASE),0) -+BACKEND += \ -+ backend/gpu/mali_kbase_pm_ca_random.c \ -+ backend/gpu/mali_kbase_pm_demand_always_powered.c \ -+ backend/gpu/mali_kbase_pm_fast_start.c -+endif -+ -+ifeq ($(CONFIG_MALI_DEVFREQ),y) -+BACKEND += \ -+ backend/gpu/mali_kbase_devfreq.c \ -+ backend/gpu/mali_kbase_pm_ca_devfreq.c -+endif -+ -+ifeq ($(CONFIG_MALI_NO_MALI),y) -+ # Dummy model -+ BACKEND += backend/gpu/mali_kbase_model_dummy.c -+ BACKEND += backend/gpu/mali_kbase_model_linux.c -+ # HW error simulation -+ BACKEND += backend/gpu/mali_kbase_model_error_generator.c -+endif -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h -new file mode 100755 -index 000000000..c8ae87eb8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h -@@ -0,0 +1,29 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Backend specific configuration -+ */ -+ -+#ifndef _KBASE_BACKEND_CONFIG_H_ -+#define _KBASE_BACKEND_CONFIG_H_ -+ -+/* Enable GPU reset API */ -+#define KBASE_GPU_RESET_EN 1 -+ -+#endif /* _KBASE_BACKEND_CONFIG_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c -new file mode 100755 -index 000000000..fef9a2cb7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c -@@ -0,0 +1,29 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include "backend/gpu/mali_kbase_cache_policy_backend.h" -+#include -+ -+void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, -+ u32 mode) -+{ -+ kbdev->current_gpu_coherency_mode = mode; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) -+ kbase_reg_write(kbdev, COHERENCY_ENABLE, mode, NULL); -+} -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h -new file mode 100755 -index 000000000..fe9869109 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h -@@ -0,0 +1,34 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+#ifndef _KBASE_CACHE_POLICY_BACKEND_H_ -+#define _KBASE_CACHE_POLICY_BACKEND_H_ -+ -+#include "mali_kbase.h" -+#include "mali_base_kernel.h" -+ -+/** -+ * kbase_cache_set_coherency_mode() - Sets the system coherency mode -+ * in the GPU. -+ * @kbdev: Device pointer -+ * @mode: Coherency mode. COHERENCY_ACE/ACE_LITE -+ */ -+void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, -+ u32 mode); -+ -+#endif /* _KBASE_CACHE_POLICY_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c -new file mode 100755 -index 000000000..7851ea646 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c -@@ -0,0 +1,157 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include "mali_kbase_debug_job_fault.h" -+ -+#ifdef CONFIG_DEBUG_FS -+ -+/*GPU_CONTROL_REG(r)*/ -+static int gpu_control_reg_snapshot[] = { -+ GPU_ID, -+ SHADER_READY_LO, -+ SHADER_READY_HI, -+ TILER_READY_LO, -+ TILER_READY_HI, -+ L2_READY_LO, -+ L2_READY_HI -+}; -+ -+/* JOB_CONTROL_REG(r) */ -+static int job_control_reg_snapshot[] = { -+ JOB_IRQ_MASK, -+ JOB_IRQ_STATUS -+}; -+ -+/* JOB_SLOT_REG(n,r) */ -+static int job_slot_reg_snapshot[] = { -+ JS_HEAD_LO, -+ JS_HEAD_HI, -+ JS_TAIL_LO, -+ JS_TAIL_HI, -+ JS_AFFINITY_LO, -+ JS_AFFINITY_HI, -+ JS_CONFIG, -+ JS_STATUS, -+ JS_HEAD_NEXT_LO, -+ JS_HEAD_NEXT_HI, -+ JS_AFFINITY_NEXT_LO, -+ JS_AFFINITY_NEXT_HI, -+ JS_CONFIG_NEXT -+}; -+ -+/*MMU_REG(r)*/ -+static int mmu_reg_snapshot[] = { -+ MMU_IRQ_MASK, -+ MMU_IRQ_STATUS -+}; -+ -+/* MMU_AS_REG(n,r) */ -+static int as_reg_snapshot[] = { -+ AS_TRANSTAB_LO, -+ AS_TRANSTAB_HI, -+ AS_MEMATTR_LO, -+ AS_MEMATTR_HI, -+ AS_FAULTSTATUS, -+ AS_FAULTADDRESS_LO, -+ AS_FAULTADDRESS_HI, -+ AS_STATUS -+}; -+ -+bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, -+ int reg_range) -+{ -+ int i, j; -+ int offset = 0; -+ int slot_number; -+ int as_number; -+ -+ if (kctx->reg_dump == NULL) -+ return false; -+ -+ slot_number = kctx->kbdev->gpu_props.num_job_slots; -+ as_number = kctx->kbdev->gpu_props.num_address_spaces; -+ -+ /* get the GPU control registers*/ -+ for (i = 0; i < sizeof(gpu_control_reg_snapshot)/4; i++) { -+ kctx->reg_dump[offset] = -+ GPU_CONTROL_REG(gpu_control_reg_snapshot[i]); -+ offset += 2; -+ } -+ -+ /* get the Job control registers*/ -+ for (i = 0; i < sizeof(job_control_reg_snapshot)/4; i++) { -+ kctx->reg_dump[offset] = -+ JOB_CONTROL_REG(job_control_reg_snapshot[i]); -+ offset += 2; -+ } -+ -+ /* get the Job Slot registers*/ -+ for (j = 0; j < slot_number; j++) { -+ for (i = 0; i < sizeof(job_slot_reg_snapshot)/4; i++) { -+ kctx->reg_dump[offset] = -+ JOB_SLOT_REG(j, job_slot_reg_snapshot[i]); -+ offset += 2; -+ } -+ } -+ -+ /* get the MMU registers*/ -+ for (i = 0; i < sizeof(mmu_reg_snapshot)/4; i++) { -+ kctx->reg_dump[offset] = MMU_REG(mmu_reg_snapshot[i]); -+ offset += 2; -+ } -+ -+ /* get the Address space registers*/ -+ for (j = 0; j < as_number; j++) { -+ for (i = 0; i < sizeof(as_reg_snapshot)/4; i++) { -+ kctx->reg_dump[offset] = -+ MMU_AS_REG(j, as_reg_snapshot[i]); -+ offset += 2; -+ } -+ } -+ -+ WARN_ON(offset >= (reg_range*2/4)); -+ -+ /* set the termination flag*/ -+ kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; -+ kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; -+ -+ dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", -+ offset); -+ -+ return true; -+} -+ -+bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) -+{ -+ int offset = 0; -+ -+ if (kctx->reg_dump == NULL) -+ return false; -+ -+ while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { -+ kctx->reg_dump[offset+1] = -+ kbase_reg_read(kctx->kbdev, -+ kctx->reg_dump[offset], NULL); -+ offset += 2; -+ } -+ return true; -+} -+ -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c -new file mode 100755 -index 000000000..ab14bc2e2 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c -@@ -0,0 +1,458 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+#define ENABLE_DEBUG_LOG -+#include "../../platform/rk/custom_log.h" -+ -+ -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#ifdef CONFIG_DEVFREQ_THERMAL -+#include -+#endif -+ -+#include -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) -+#include -+#else /* Linux >= 3.13 */ -+/* In 3.13 the OPP include header file, types, and functions were all -+ * renamed. Use the old filename for the include, and define the new names to -+ * the old, when an old kernel is detected. -+ */ -+#include -+#define dev_pm_opp opp -+#define dev_pm_opp_get_voltage opp_get_voltage -+#define dev_pm_opp_get_opp_count opp_get_opp_count -+#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil -+#define dev_pm_opp_find_freq_floor opp_find_freq_floor -+#endif /* Linux >= 3.13 */ -+#include -+#include -+ -+static struct devfreq_simple_ondemand_data ondemand_data; -+ -+static struct monitor_dev_profile mali_mdevp = { -+ .type = MONITOR_TPYE_DEV, -+ .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, -+ .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, -+}; -+ -+/** -+ * opp_translate - Translate nominal OPP frequency from devicetree into real -+ * frequency and core mask -+ * @kbdev: Device pointer -+ * @freq: Nominal frequency -+ * @core_mask: Pointer to u64 to store core mask to -+ * -+ * Return: Real target frequency -+ * -+ * This function will only perform translation if an operating-points-v2-mali -+ * table is present in devicetree. If one is not present then it will return an -+ * untranslated frequency and all cores enabled. -+ */ -+static unsigned long opp_translate(struct kbase_device *kbdev, -+ unsigned long freq, u64 *core_mask) -+{ -+ int i; -+ -+ for (i = 0; i < kbdev->num_opps; i++) { -+ if (kbdev->opp_table[i].opp_freq == freq) { -+ *core_mask = kbdev->opp_table[i].core_mask; -+ return kbdev->opp_table[i].real_freq; -+ } -+ } -+ -+ /* Failed to find OPP - return all cores enabled & nominal frequency */ -+ *core_mask = kbdev->gpu_props.props.raw_props.shader_present; -+ -+ return freq; -+} -+ -+static int -+kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct dev_pm_opp *opp; -+ unsigned long nominal_freq; -+ unsigned long freq = 0; -+ unsigned long old_freq = kbdev->current_freq; -+ unsigned long voltage; -+ int err; -+ u64 core_mask; -+ -+ freq = *target_freq; -+ -+ opp = devfreq_recommended_opp(dev, &freq, flags); -+ if (IS_ERR(opp)) { -+ dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); -+ return PTR_ERR(opp); -+ } -+ voltage = dev_pm_opp_get_voltage(opp); -+ -+ nominal_freq = freq; -+ -+ /* -+ * Only update if there is a change of frequency -+ */ -+ if (kbdev->current_nominal_freq == nominal_freq) { -+ *target_freq = nominal_freq; -+#ifdef CONFIG_REGULATOR -+ if (kbdev->current_voltage == voltage) -+ return 0; -+ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); -+ if (err) { -+ dev_err(dev, "Failed to set voltage (%d)\n", err); -+ return err; -+ } -+ kbdev->current_voltage = voltage; -+#endif -+ return 0; -+ } -+ -+ freq = opp_translate(kbdev, nominal_freq, &core_mask); -+#ifdef CONFIG_REGULATOR -+ if (kbdev->regulator && kbdev->current_voltage != voltage && -+ old_freq < freq) { -+ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); -+ if (err) { -+ dev_err(dev, "Failed to increase voltage (%d)\n", err); -+ return err; -+ } -+ } -+#endif -+ -+ err = clk_set_rate(kbdev->clock, freq); -+ if (err) { -+ dev_err(dev, "Failed to set clock %lu (target %lu)\n", -+ freq, *target_freq); -+ return err; -+ } -+ *target_freq = freq; -+ kbdev->current_freq = freq; -+ if (kbdev->devfreq) -+ kbdev->devfreq->last_status.current_frequency = freq; -+#ifdef CONFIG_REGULATOR -+ if (kbdev->regulator && kbdev->current_voltage != voltage && -+ old_freq > freq) { -+ err = regulator_set_voltage(kbdev->regulator, voltage, INT_MAX); -+ if (err) { -+ dev_err(dev, "Failed to decrease voltage (%d)\n", err); -+ return err; -+ } -+ } -+#endif -+ -+ if (kbdev->pm.backend.ca_current_policy->id == -+ KBASE_PM_CA_POLICY_ID_DEVFREQ) -+ kbase_devfreq_set_core_mask(kbdev, core_mask); -+ -+ *target_freq = nominal_freq; -+ kbdev->current_voltage = voltage; -+ kbdev->current_nominal_freq = nominal_freq; -+ kbdev->current_freq = freq; -+ kbdev->current_core_mask = core_mask; -+ -+ KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq); -+ -+ kbase_pm_reset_dvfs_utilisation(kbdev); -+ -+ return err; -+} -+ -+static int -+kbase_devfreq_cur_freq(struct device *dev, unsigned long *freq) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ -+ *freq = kbdev->current_nominal_freq; -+ -+ return 0; -+} -+ -+static int -+kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ -+ stat->current_frequency = kbdev->current_nominal_freq; -+ -+ kbase_pm_get_dvfs_utilisation(kbdev, -+ &stat->total_time, &stat->busy_time); -+ -+ stat->private_data = NULL; -+ -+ return 0; -+} -+ -+static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, -+ struct devfreq_dev_profile *dp) -+{ -+ int count; -+ int i = 0; -+ unsigned long freq; -+ struct dev_pm_opp *opp; -+ -+ count = dev_pm_opp_get_opp_count(kbdev->dev); -+ if (count < 0) { -+ return count; -+ } -+ -+ dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), -+ GFP_KERNEL); -+ if (!dp->freq_table) -+ return -ENOMEM; -+ -+ for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { -+ opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); -+ if (IS_ERR(opp)) -+ break; -+ dev_pm_opp_put(opp); -+ -+ dp->freq_table[i] = freq; -+ } -+ -+ if (count != i) -+ dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", -+ count, i); -+ -+ dp->max_state = i; -+ -+ return 0; -+} -+ -+static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) -+{ -+ struct devfreq_dev_profile *dp = kbdev->devfreq->profile; -+ -+ kfree(dp->freq_table); -+} -+ -+static void kbase_devfreq_exit(struct device *dev) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ -+ kbase_devfreq_term_freq_table(kbdev); -+} -+ -+static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) -+{ -+ struct device_node *opp_node = of_parse_phandle(kbdev->dev->of_node, -+ "operating-points-v2", 0); -+ struct device_node *node; -+ int i = 0; -+ int count; -+ -+ if (!opp_node) -+ return 0; -+ if (!of_device_is_compatible(opp_node, "operating-points-v2-mali")) -+ return 0; -+ -+ count = dev_pm_opp_get_opp_count(kbdev->dev); -+ kbdev->opp_table = kmalloc_array(count, -+ sizeof(struct kbase_devfreq_opp), GFP_KERNEL); -+ if (!kbdev->opp_table) -+ return -ENOMEM; -+ -+ for_each_available_child_of_node(opp_node, node) { -+ u64 core_mask; -+ u64 opp_freq, real_freq; -+ const void *core_count_p; -+ -+ if (of_property_read_u64(node, "opp-hz", &opp_freq)) { -+ dev_warn(kbdev->dev, "OPP is missing required opp-hz property\n"); -+ continue; -+ } -+ if (of_property_read_u64(node, "opp-hz-real", &real_freq)) -+ real_freq = opp_freq; -+ if (of_property_read_u64(node, "opp-core-mask", &core_mask)) -+ core_mask = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ core_count_p = of_get_property(node, "opp-core-count", NULL); -+ if (core_count_p) { -+ u64 remaining_core_mask = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ int core_count = be32_to_cpup(core_count_p); -+ -+ core_mask = 0; -+ -+ for (; core_count > 0; core_count--) { -+ int core = ffs(remaining_core_mask); -+ -+ if (!core) { -+ dev_err(kbdev->dev, "OPP has more cores than GPU\n"); -+ return -ENODEV; -+ } -+ -+ core_mask |= (1ull << (core-1)); -+ remaining_core_mask &= ~(1ull << (core-1)); -+ } -+ } -+ -+ if (!core_mask) { -+ dev_err(kbdev->dev, "OPP has invalid core mask of 0\n"); -+ return -ENODEV; -+ } -+ -+ kbdev->opp_table[i].opp_freq = opp_freq; -+ kbdev->opp_table[i].real_freq = real_freq; -+ kbdev->opp_table[i].core_mask = core_mask; -+ -+ dev_info(kbdev->dev, "OPP %d : opp_freq=%llu real_freq=%llu core_mask=%llx\n", -+ i, opp_freq, real_freq, core_mask); -+ -+ i++; -+ } -+ -+ kbdev->num_opps = i; -+ -+ return 0; -+} -+ -+int kbase_devfreq_init(struct kbase_device *kbdev) -+{ -+ struct device_node *np = kbdev->dev->of_node; -+ struct devfreq_dev_profile *dp; -+ struct dev_pm_opp *opp; -+ unsigned long opp_rate; -+ int err; -+ -+ if (!kbdev->clock) { -+ dev_err(kbdev->dev, "Clock not available for devfreq\n"); -+ return -ENODEV; -+ } -+ -+ kbdev->current_freq = clk_get_rate(kbdev->clock); -+ kbdev->current_nominal_freq = kbdev->current_freq; -+ -+ dp = &kbdev->devfreq_profile; -+ -+ dp->initial_freq = kbdev->current_freq; -+ /* .KP : set devfreq_dvfs_interval_in_ms */ -+ dp->polling_ms = 20; -+ dp->target = kbase_devfreq_target; -+ dp->get_dev_status = kbase_devfreq_status; -+ dp->get_cur_freq = kbase_devfreq_cur_freq; -+ dp->exit = kbase_devfreq_exit; -+ -+ if (kbase_devfreq_init_freq_table(kbdev, dp)) -+ return -EFAULT; -+ -+ err = kbase_devfreq_init_core_mask_table(kbdev); -+ if (err) -+ return err; -+ -+ of_property_read_u32(np, "upthreshold", -+ &ondemand_data.upthreshold); -+ of_property_read_u32(np, "downdifferential", -+ &ondemand_data.downdifferential); -+ -+ kbdev->devfreq = devfreq_add_device(kbdev->dev, dp, -+ "simple_ondemand", &ondemand_data); -+ if (IS_ERR(kbdev->devfreq)) { -+ kbase_devfreq_term_freq_table(kbdev); -+ return PTR_ERR(kbdev->devfreq); -+ } -+ -+ /* devfreq_add_device only copies a few of kbdev->dev's fields, so -+ * set drvdata explicitly so IPA models can access kbdev. */ -+ dev_set_drvdata(&kbdev->devfreq->dev, kbdev); -+ -+ err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); -+ if (err) { -+ dev_err(kbdev->dev, -+ "Failed to register OPP notifier (%d)\n", err); -+ goto opp_notifier_failed; -+ } -+ -+ opp_rate = kbdev->current_freq; -+ opp = devfreq_recommended_opp(kbdev->dev, &opp_rate, 0); -+ if (!IS_ERR(opp)) -+ dev_pm_opp_put(opp); -+ kbdev->devfreq->last_status.current_frequency = opp_rate; -+ -+ mali_mdevp.data = kbdev->devfreq; -+ kbdev->mdev_info = rockchip_system_monitor_register(kbdev->dev, -+ &mali_mdevp); -+ if (IS_ERR(kbdev->mdev_info)) { -+ dev_dbg(kbdev->dev, "without system monitor\n"); -+ kbdev->mdev_info = NULL; -+ } -+#ifdef CONFIG_DEVFREQ_THERMAL -+ err = kbase_ipa_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "IPA initialization failed\n"); -+ goto cooling_failed; -+ } -+ -+ kbdev->devfreq_cooling = of_devfreq_cooling_register_power( -+ kbdev->dev->of_node, -+ kbdev->devfreq, -+ &kbase_ipa_power_model_ops); -+ if (IS_ERR_OR_NULL(kbdev->devfreq_cooling)) { -+ err = PTR_ERR(kbdev->devfreq_cooling); -+ dev_err(kbdev->dev, -+ "Failed to register cooling device (%d)\n", -+ err); -+ goto cooling_failed; -+ } -+ I("success initing power_model_simple."); -+#endif -+ -+ return 0; -+ -+#ifdef CONFIG_DEVFREQ_THERMAL -+cooling_failed: -+ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); -+#endif /* CONFIG_DEVFREQ_THERMAL */ -+opp_notifier_failed: -+ if (devfreq_remove_device(kbdev->devfreq)) -+ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); -+ else -+ kbdev->devfreq = NULL; -+ -+ return err; -+} -+ -+void kbase_devfreq_term(struct kbase_device *kbdev) -+{ -+ int err; -+ -+ dev_dbg(kbdev->dev, "Term Mali devfreq\n"); -+ -+ rockchip_system_monitor_unregister(kbdev->mdev_info); -+#ifdef CONFIG_DEVFREQ_THERMAL -+ if (kbdev->devfreq_cooling) -+ devfreq_cooling_unregister(kbdev->devfreq_cooling); -+ -+ kbase_ipa_term(kbdev); -+#endif -+ -+ devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); -+ -+ err = devfreq_remove_device(kbdev->devfreq); -+ if (err) -+ dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); -+ else -+ kbdev->devfreq = NULL; -+ -+ kfree(kbdev->opp_table); -+} -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h -new file mode 100755 -index 000000000..c0bf8b15b ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h -@@ -0,0 +1,24 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _BASE_DEVFREQ_H_ -+#define _BASE_DEVFREQ_H_ -+ -+int kbase_devfreq_init(struct kbase_device *kbdev); -+void kbase_devfreq_term(struct kbase_device *kbdev); -+ -+#endif /* _BASE_DEVFREQ_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c -new file mode 100755 -index 000000000..dcdf15cdc ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c -@@ -0,0 +1,255 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * -+ */ -+#include -+#include -+#include -+ -+#include -+ -+#if !defined(CONFIG_MALI_NO_MALI) -+ -+ -+#ifdef CONFIG_DEBUG_FS -+ -+ -+int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) -+{ -+ struct kbase_io_access *old_buf; -+ struct kbase_io_access *new_buf; -+ unsigned long flags; -+ -+ if (!new_size) -+ goto out_err; /* The new size must not be 0 */ -+ -+ new_buf = vmalloc(new_size * sizeof(*h->buf)); -+ if (!new_buf) -+ goto out_err; -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ old_buf = h->buf; -+ -+ /* Note: we won't bother with copying the old data over. The dumping -+ * logic wouldn't work properly as it relies on 'count' both as a -+ * counter and as an index to the buffer which would have changed with -+ * the new array. This is a corner case that we don't need to support. -+ */ -+ h->count = 0; -+ h->size = new_size; -+ h->buf = new_buf; -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+ -+ vfree(old_buf); -+ -+ return 0; -+ -+out_err: -+ return -1; -+} -+ -+ -+int kbase_io_history_init(struct kbase_io_history *h, u16 n) -+{ -+ h->enabled = false; -+ spin_lock_init(&h->lock); -+ h->count = 0; -+ h->size = 0; -+ h->buf = NULL; -+ if (kbase_io_history_resize(h, n)) -+ return -1; -+ -+ return 0; -+} -+ -+ -+void kbase_io_history_term(struct kbase_io_history *h) -+{ -+ vfree(h->buf); -+ h->buf = NULL; -+} -+ -+ -+/* kbase_io_history_add - add new entry to the register access history -+ * -+ * @h: Pointer to the history data structure -+ * @addr: Register address -+ * @value: The value that is either read from or written to the register -+ * @write: 1 if it's a register write, 0 if it's a read -+ */ -+static void kbase_io_history_add(struct kbase_io_history *h, -+ void __iomem const *addr, u32 value, u8 write) -+{ -+ struct kbase_io_access *io; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ io = &h->buf[h->count % h->size]; -+ io->addr = (uintptr_t)addr | write; -+ io->value = value; -+ ++h->count; -+ /* If count overflows, move the index by the buffer size so the entire -+ * buffer will still be dumped later */ -+ if (unlikely(!h->count)) -+ h->count = h->size; -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+} -+ -+ -+void kbase_io_history_dump(struct kbase_device *kbdev) -+{ -+ struct kbase_io_history *const h = &kbdev->io_history; -+ u16 i; -+ size_t iters; -+ unsigned long flags; -+ -+ if (!unlikely(h->enabled)) -+ return; -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ dev_err(kbdev->dev, "Register IO History:"); -+ iters = (h->size > h->count) ? h->count : h->size; -+ dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, -+ h->count); -+ for (i = 0; i < iters; ++i) { -+ struct kbase_io_access *io = -+ &h->buf[(h->count - iters + i) % h->size]; -+ char const access = (io->addr & 1) ? 'w' : 'r'; -+ -+ dev_err(kbdev->dev, "%6i: %c: reg 0x%p val %08x\n", i, access, -+ (void *)(io->addr & ~0x1), io->value); -+ } -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+} -+ -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+ -+void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, -+ struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -+ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); -+ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); -+ -+ writel(value, kbdev->reg + offset); -+ -+#ifdef CONFIG_DEBUG_FS -+ if (unlikely(kbdev->io_history.enabled)) -+ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, -+ value, 1); -+#endif /* CONFIG_DEBUG_FS */ -+ dev_dbg(kbdev->dev, "w: reg %04x val %08x", offset, value); -+ -+ if (kctx && kctx->jctx.tb) -+ kbase_device_trace_register_access(kctx, REG_WRITE, offset, -+ value); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_reg_write); -+ -+u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, -+ struct kbase_context *kctx) -+{ -+ u32 val; -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -+ KBASE_DEBUG_ASSERT(kctx == NULL || kctx->as_nr != KBASEP_AS_NR_INVALID); -+ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); -+ -+ val = readl(kbdev->reg + offset); -+ -+#ifdef CONFIG_DEBUG_FS -+ if (unlikely(kbdev->io_history.enabled)) -+ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, -+ val, 0); -+#endif /* CONFIG_DEBUG_FS */ -+ dev_dbg(kbdev->dev, "r: reg %04x val %08x", offset, val); -+ -+ if (kctx && kctx->jctx.tb) -+ kbase_device_trace_register_access(kctx, REG_READ, offset, val); -+ return val; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_reg_read); -+#endif /* !defined(CONFIG_MALI_NO_MALI) */ -+ -+/** -+ * kbase_report_gpu_fault - Report a GPU fault. -+ * @kbdev: Kbase device pointer -+ * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS -+ * was also set -+ * -+ * This function is called from the interrupt handler when a GPU fault occurs. -+ * It reports the details of the fault using dev_warn(). -+ */ -+static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) -+{ -+ u32 status; -+ u64 address; -+ -+ status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL); -+ address = (u64) kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI), NULL) << 32; -+ address |= kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO), NULL); -+ -+ dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", -+ status & 0xFF, -+ kbase_exception_name(kbdev, status), -+ address); -+ if (multiple) -+ dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); -+} -+ -+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) -+{ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val); -+ if (val & GPU_FAULT) -+ kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); -+ -+ if (val & RESET_COMPLETED) -+ kbase_pm_reset_done(kbdev); -+ -+ if (val & PRFCNT_SAMPLE_COMPLETED) -+ kbase_instr_hwcnt_sample_done(kbdev); -+ -+ if (val & CLEAN_CACHES_COMPLETED) -+ kbase_clean_caches_done(kbdev); -+ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, NULL); -+ -+ /* kbase_pm_check_transitions must be called after the IRQ has been -+ * cleared. This is because it might trigger further power transitions -+ * and we don't want to miss the interrupt raised to notify us that -+ * these further transitions have finished. -+ */ -+ if (val & POWER_CHANGED_ALL) -+ kbase_pm_power_changed(kbdev); -+ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val); -+} -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h -new file mode 100755 -index 000000000..5b2044593 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h -@@ -0,0 +1,67 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Backend-specific HW access device APIs -+ */ -+ -+#ifndef _KBASE_DEVICE_INTERNAL_H_ -+#define _KBASE_DEVICE_INTERNAL_H_ -+ -+/** -+ * kbase_reg_write - write to GPU register -+ * @kbdev: Kbase device pointer -+ * @offset: Offset of register -+ * @value: Value to write -+ * @kctx: Kbase context pointer. May be NULL -+ * -+ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If -+ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr -+ * != KBASEP_AS_NR_INVALID). -+ */ -+void kbase_reg_write(struct kbase_device *kbdev, u16 offset, u32 value, -+ struct kbase_context *kctx); -+ -+/** -+ * kbase_reg_read - read from GPU register -+ * @kbdev: Kbase device pointer -+ * @offset: Offset of register -+ * @kctx: Kbase context pointer. May be NULL -+ * -+ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). If -+ * @kctx is not NULL then the caller must ensure it is scheduled (@kctx->as_nr -+ * != KBASEP_AS_NR_INVALID). -+ * -+ * Return: Value in desired register -+ */ -+u32 kbase_reg_read(struct kbase_device *kbdev, u16 offset, -+ struct kbase_context *kctx); -+ -+ -+/** -+ * kbase_gpu_interrupt - GPU interrupt handler -+ * @kbdev: Kbase device pointer -+ * @val: The value of the GPU IRQ status register which triggered the call -+ * -+ * This function is called from the interrupt handler when a GPU irq is to be -+ * handled. -+ */ -+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); -+ -+#endif /* _KBASE_DEVICE_INTERNAL_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c -new file mode 100755 -index 000000000..d578fd78e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpu.c -@@ -0,0 +1,123 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register-based HW access backend APIs -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+int kbase_backend_early_init(struct kbase_device *kbdev) -+{ -+ int err; -+ -+ err = kbasep_platform_device_init(kbdev); -+ if (err) -+ return err; -+ -+ /* Ensure we can access the GPU registers */ -+ kbase_pm_register_access_enable(kbdev); -+ -+ /* Find out GPU properties based on the GPU feature registers */ -+ kbase_gpuprops_set(kbdev); -+ -+ /* We're done accessing the GPU registers for now. */ -+ kbase_pm_register_access_disable(kbdev); -+ -+ err = kbase_hwaccess_pm_init(kbdev); -+ if (err) -+ goto fail_pm; -+ -+ err = kbase_install_interrupts(kbdev); -+ if (err) -+ goto fail_interrupts; -+ -+ return 0; -+ -+fail_interrupts: -+ kbase_hwaccess_pm_term(kbdev); -+fail_pm: -+ kbasep_platform_device_term(kbdev); -+ -+ return err; -+} -+ -+void kbase_backend_early_term(struct kbase_device *kbdev) -+{ -+ kbase_release_interrupts(kbdev); -+ kbase_hwaccess_pm_term(kbdev); -+ kbasep_platform_device_term(kbdev); -+} -+ -+int kbase_backend_late_init(struct kbase_device *kbdev) -+{ -+ int err; -+ -+ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); -+ if (err) -+ return err; -+ -+ err = kbase_backend_timer_init(kbdev); -+ if (err) -+ goto fail_timer; -+ -+#ifdef CONFIG_MALI_DEBUG -+#ifndef CONFIG_MALI_NO_MALI -+ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { -+ dev_err(kbdev->dev, "Interrupt assigment check failed.\n"); -+ err = -EINVAL; -+ goto fail_interrupt_test; -+ } -+#endif /* !CONFIG_MALI_NO_MALI */ -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ err = kbase_job_slot_init(kbdev); -+ if (err) -+ goto fail_job_slot; -+ -+ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); -+ -+ return 0; -+ -+fail_job_slot: -+ -+#ifdef CONFIG_MALI_DEBUG -+#ifndef CONFIG_MALI_NO_MALI -+fail_interrupt_test: -+#endif /* !CONFIG_MALI_NO_MALI */ -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ kbase_backend_timer_term(kbdev); -+fail_timer: -+ kbase_hwaccess_pm_halt(kbdev); -+ -+ return err; -+} -+ -+void kbase_backend_late_term(struct kbase_device *kbdev) -+{ -+ kbase_job_slot_halt(kbdev); -+ kbase_job_slot_term(kbdev); -+ kbase_backend_timer_term(kbdev); -+ kbase_hwaccess_pm_halt(kbdev); -+} -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c -new file mode 100755 -index 000000000..b395325b5 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c -@@ -0,0 +1,110 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Base kernel property query backend APIs -+ */ -+ -+#include -+#include -+#include -+#include -+ -+void kbase_backend_gpuprops_get(struct kbase_device *kbdev, -+ struct kbase_gpuprops_regdump *regdump) -+{ -+ int i; -+ -+ /* Fill regdump with the content of the relevant registers */ -+ regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID), NULL); -+ -+ regdump->l2_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_FEATURES), NULL); -+ regdump->suspend_size = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SUSPEND_SIZE), NULL); -+ regdump->tiler_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TILER_FEATURES), NULL); -+ regdump->mem_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(MEM_FEATURES), NULL); -+ regdump->mmu_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(MMU_FEATURES), NULL); -+ regdump->as_present = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(AS_PRESENT), NULL); -+ regdump->js_present = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(JS_PRESENT), NULL); -+ -+ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) -+ regdump->js_features[i] = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(JS_FEATURES_REG(i)), NULL); -+ -+ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) -+ regdump->texture_features[i] = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)), NULL); -+ -+ regdump->thread_max_threads = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(THREAD_MAX_THREADS), NULL); -+ regdump->thread_max_workgroup_size = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE), -+ NULL); -+ regdump->thread_max_barrier_size = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE), NULL); -+ regdump->thread_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(THREAD_FEATURES), NULL); -+ -+ regdump->shader_present_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_PRESENT_LO), NULL); -+ regdump->shader_present_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_PRESENT_HI), NULL); -+ -+ regdump->tiler_present_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TILER_PRESENT_LO), NULL); -+ regdump->tiler_present_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TILER_PRESENT_HI), NULL); -+ -+ regdump->l2_present_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_PRESENT_LO), NULL); -+ regdump->l2_present_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_PRESENT_HI), NULL); -+ -+ regdump->stack_present_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(STACK_PRESENT_LO), NULL); -+ regdump->stack_present_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(STACK_PRESENT_HI), NULL); -+} -+ -+void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, -+ struct kbase_gpuprops_regdump *regdump) -+{ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) { -+ /* Ensure we can access the GPU registers */ -+ kbase_pm_register_access_enable(kbdev); -+ -+ regdump->coherency_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); -+ -+ /* We're done accessing the GPU registers for now. */ -+ kbase_pm_register_access_disable(kbdev); -+ } else { -+ /* Pre COHERENCY_FEATURES we only supported ACE_LITE */ -+ regdump->coherency_features = -+ COHERENCY_FEATURE_BIT(COHERENCY_NONE) | -+ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); -+ } -+} -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c -new file mode 100755 -index 000000000..7ad309e8d ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c -@@ -0,0 +1,492 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * GPU backend instrumentation APIs. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/** -+ * kbasep_instr_hwcnt_cacheclean - Issue Cache Clean & Invalidate command to -+ * hardware -+ * -+ * @kbdev: Kbase device -+ */ -+static void kbasep_instr_hwcnt_cacheclean(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ unsigned long pm_flags; -+ u32 irq_mask; -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == -+ KBASE_INSTR_STATE_REQUEST_CLEAN); -+ -+ /* Enable interrupt */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); -+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ irq_mask | CLEAN_CACHES_COMPLETED, NULL); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); -+ -+ /* clean&invalidate the caches so we're sure the mmu tables for the dump -+ * buffer is valid */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAN_INV_CACHES, NULL); -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANING; -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+} -+ -+int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ struct kbase_uk_hwcnt_setup *setup) -+{ -+ unsigned long flags, pm_flags; -+ int err = -EINVAL; -+ u32 irq_mask; -+ int ret; -+ u64 shader_cores_needed; -+ u32 prfcnt_config; -+ -+ shader_cores_needed = kbase_pm_get_present_cores(kbdev, -+ KBASE_PM_CORE_SHADER); -+ -+ /* alignment failure */ -+ if ((setup->dump_buffer == 0ULL) || (setup->dump_buffer & (2048 - 1))) -+ goto out_err; -+ -+ /* Override core availability policy to ensure all cores are available -+ */ -+ kbase_pm_ca_instr_enable(kbdev); -+ -+ /* Request the cores early on synchronously - we'll release them on any -+ * errors (e.g. instrumentation already active) */ -+ kbase_pm_request_cores_sync(kbdev, true, shader_cores_needed); -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { -+ /* Instrumentation is already enabled */ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ goto out_unrequest_cores; -+ } -+ -+ /* Enable interrupt */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); -+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), irq_mask | -+ PRFCNT_SAMPLE_COMPLETED, NULL); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); -+ -+ /* In use, this context is the owner */ -+ kbdev->hwcnt.kctx = kctx; -+ /* Remember the dump address so we can reprogram it later */ -+ kbdev->hwcnt.addr = setup->dump_buffer; -+ -+ /* Request the clean */ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; -+ kbdev->hwcnt.backend.triggered = 0; -+ /* Clean&invalidate the caches so we're sure the mmu tables for the dump -+ * buffer is valid */ -+ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, -+ &kbdev->hwcnt.backend.cache_clean_work); -+ KBASE_DEBUG_ASSERT(ret); -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ /* Wait for cacheclean to complete */ -+ wait_event(kbdev->hwcnt.backend.wait, -+ kbdev->hwcnt.backend.triggered != 0); -+ -+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == -+ KBASE_INSTR_STATE_IDLE); -+ -+ kbase_pm_request_l2_caches(kbdev); -+ -+ /* Configure */ -+ prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; -+#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY -+ { -+ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ u32 product_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) -+ >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ int arch_v6 = GPU_ID_IS_NEW_FORMAT(product_id); -+ -+ if (arch_v6) -+ prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; -+ } -+#endif -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), -+ prfcnt_config | PRFCNT_CONFIG_MODE_OFF, kctx); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), -+ setup->dump_buffer & 0xFFFFFFFF, kctx); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), -+ setup->dump_buffer >> 32, kctx); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), -+ setup->jm_bm, kctx); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), -+ setup->shader_bm, kctx); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), -+ setup->mmu_l2_bm, kctx); -+ /* Due to PRLAM-8186 we need to disable the Tiler before we enable the -+ * HW counter dump. */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, -+ kctx); -+ else -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), -+ setup->tiler_bm, kctx); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), -+ prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL, kctx); -+ -+ /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump -+ */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), -+ setup->tiler_bm, kctx); -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -+ kbdev->hwcnt.backend.triggered = 1; -+ wake_up(&kbdev->hwcnt.backend.wait); -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ err = 0; -+ -+ dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); -+ return err; -+ out_unrequest_cores: -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_pm_unrequest_cores(kbdev, true, shader_cores_needed); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ out_err: -+ return err; -+} -+ -+int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) -+{ -+ unsigned long flags, pm_flags; -+ int err = -EINVAL; -+ u32 irq_mask; -+ struct kbase_device *kbdev = kctx->kbdev; -+ -+ while (1) { -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { -+ /* Instrumentation is not enabled */ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ goto out; -+ } -+ -+ if (kbdev->hwcnt.kctx != kctx) { -+ /* Instrumentation has been setup for another context */ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ goto out; -+ } -+ -+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) -+ break; -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ /* Ongoing dump/setup - wait for its completion */ -+ wait_event(kbdev->hwcnt.backend.wait, -+ kbdev->hwcnt.backend.triggered != 0); -+ } -+ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; -+ kbdev->hwcnt.backend.triggered = 0; -+ -+ /* Disable interrupt */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); -+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ irq_mask & ~PRFCNT_SAMPLE_COMPLETED, NULL); -+ -+ /* Disable the counters */ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), 0, kctx); -+ -+ kbdev->hwcnt.kctx = NULL; -+ kbdev->hwcnt.addr = 0ULL; -+ -+ kbase_pm_ca_instr_disable(kbdev); -+ -+ kbase_pm_unrequest_cores(kbdev, true, -+ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); -+ -+ kbase_pm_release_l2_caches(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", -+ kctx); -+ -+ err = 0; -+ -+ out: -+ return err; -+} -+ -+int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) -+{ -+ unsigned long flags; -+ int err = -EINVAL; -+ struct kbase_device *kbdev = kctx->kbdev; -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ if (kbdev->hwcnt.kctx != kctx) { -+ /* The instrumentation has been setup for another context */ -+ goto unlock; -+ } -+ -+ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { -+ /* HW counters are disabled or another dump is ongoing, or we're -+ * resetting */ -+ goto unlock; -+ } -+ -+ kbdev->hwcnt.backend.triggered = 0; -+ -+ /* Mark that we're dumping - the PF handler can signal that we faulted -+ */ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; -+ -+ /* Reconfigure the dump address */ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), -+ kbdev->hwcnt.addr & 0xFFFFFFFF, NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), -+ kbdev->hwcnt.addr >> 32, NULL); -+ -+ /* Start dumping */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL, -+ kbdev->hwcnt.addr, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_PRFCNT_SAMPLE, kctx); -+ -+ dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); -+ -+ err = 0; -+ -+ unlock: -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ return err; -+} -+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); -+ -+bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, -+ bool * const success) -+{ -+ unsigned long flags; -+ bool complete = false; -+ struct kbase_device *kbdev = kctx->kbdev; -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { -+ *success = true; -+ complete = true; -+ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { -+ *success = false; -+ complete = true; -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ return complete; -+} -+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); -+ -+void kbasep_cache_clean_worker(struct work_struct *data) -+{ -+ struct kbase_device *kbdev; -+ unsigned long flags; -+ -+ kbdev = container_of(data, struct kbase_device, -+ hwcnt.backend.cache_clean_work); -+ -+ mutex_lock(&kbdev->cacheclean_lock); -+ kbasep_instr_hwcnt_cacheclean(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ /* Wait for our condition, and any reset to complete */ -+ while (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ wait_event(kbdev->hwcnt.backend.cache_clean_wait, -+ kbdev->hwcnt.backend.state != -+ KBASE_INSTR_STATE_CLEANING); -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ } -+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == -+ KBASE_INSTR_STATE_CLEANED); -+ -+ /* All finished and idle */ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -+ kbdev->hwcnt.backend.triggered = 1; -+ wake_up(&kbdev->hwcnt.backend.wait); -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ mutex_unlock(&kbdev->cacheclean_lock); -+} -+ -+void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { -+ kbdev->hwcnt.backend.triggered = 1; -+ wake_up(&kbdev->hwcnt.backend.wait); -+ } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { -+ int ret; -+ /* Always clean and invalidate the cache after a successful dump -+ */ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; -+ ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, -+ &kbdev->hwcnt.backend.cache_clean_work); -+ KBASE_DEBUG_ASSERT(ret); -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+} -+ -+void kbase_clean_caches_done(struct kbase_device *kbdev) -+{ -+ u32 irq_mask; -+ -+ if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { -+ unsigned long flags; -+ unsigned long pm_flags; -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ /* Disable interrupt */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); -+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ irq_mask & ~CLEAN_CACHES_COMPLETED, NULL); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); -+ -+ /* Wakeup... */ -+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_CLEANING) { -+ /* Only wake if we weren't resetting */ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED; -+ wake_up(&kbdev->hwcnt.backend.cache_clean_wait); -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ } -+} -+ -+int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ unsigned long flags; -+ int err; -+ -+ /* Wait for dump & cacheclean to complete */ -+ wait_event(kbdev->hwcnt.backend.wait, -+ kbdev->hwcnt.backend.triggered != 0); -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { -+ err = -EINVAL; -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -+ } else { -+ /* Dump done */ -+ KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == -+ KBASE_INSTR_STATE_IDLE); -+ err = 0; -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ return err; -+} -+ -+int kbase_instr_hwcnt_clear(struct kbase_context *kctx) -+{ -+ unsigned long flags; -+ int err = -EINVAL; -+ struct kbase_device *kbdev = kctx->kbdev; -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ -+ /* Check it's the context previously set up and we're not already -+ * dumping */ -+ if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != -+ KBASE_INSTR_STATE_IDLE) -+ goto out; -+ -+ /* Clear the counters */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_PRFCNT_CLEAR, kctx); -+ -+ err = 0; -+ -+out: -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ return err; -+} -+KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); -+ -+int kbase_instr_backend_init(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; -+ -+ init_waitqueue_head(&kbdev->hwcnt.backend.wait); -+ init_waitqueue_head(&kbdev->hwcnt.backend.cache_clean_wait); -+ INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, -+ kbasep_cache_clean_worker); -+ kbdev->hwcnt.backend.triggered = 0; -+ -+ kbdev->hwcnt.backend.cache_clean_wq = -+ alloc_workqueue("Mali cache cleaning workqueue", 0, 1); -+ if (NULL == kbdev->hwcnt.backend.cache_clean_wq) -+ ret = -EINVAL; -+ -+ return ret; -+} -+ -+void kbase_instr_backend_term(struct kbase_device *kbdev) -+{ -+ destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); -+} -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h -new file mode 100755 -index 000000000..4794672da ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h -@@ -0,0 +1,58 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Backend-specific instrumentation definitions -+ */ -+ -+#ifndef _KBASE_INSTR_DEFS_H_ -+#define _KBASE_INSTR_DEFS_H_ -+ -+/* -+ * Instrumentation State Machine States -+ */ -+enum kbase_instr_state { -+ /* State where instrumentation is not active */ -+ KBASE_INSTR_STATE_DISABLED = 0, -+ /* State machine is active and ready for a command. */ -+ KBASE_INSTR_STATE_IDLE, -+ /* Hardware is currently dumping a frame. */ -+ KBASE_INSTR_STATE_DUMPING, -+ /* We've requested a clean to occur on a workqueue */ -+ KBASE_INSTR_STATE_REQUEST_CLEAN, -+ /* Hardware is currently cleaning and invalidating caches. */ -+ KBASE_INSTR_STATE_CLEANING, -+ /* Cache clean completed, and either a) a dump is complete, or -+ * b) instrumentation can now be setup. */ -+ KBASE_INSTR_STATE_CLEANED, -+ /* An error has occured during DUMPING (page fault). */ -+ KBASE_INSTR_STATE_FAULT -+}; -+ -+/* Structure used for instrumentation and HW counters dumping */ -+struct kbase_instr_backend { -+ wait_queue_head_t wait; -+ int triggered; -+ -+ enum kbase_instr_state state; -+ wait_queue_head_t cache_clean_wait; -+ struct workqueue_struct *cache_clean_wq; -+ struct work_struct cache_clean_work; -+}; -+ -+#endif /* _KBASE_INSTR_DEFS_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h -new file mode 100755 -index 000000000..e96aeae78 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h -@@ -0,0 +1,45 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Backend-specific HW access instrumentation APIs -+ */ -+ -+#ifndef _KBASE_INSTR_INTERNAL_H_ -+#define _KBASE_INSTR_INTERNAL_H_ -+ -+/** -+ * kbasep_cache_clean_worker() - Workqueue for handling cache cleaning -+ * @data: a &struct work_struct -+ */ -+void kbasep_cache_clean_worker(struct work_struct *data); -+ -+/** -+ * kbase_clean_caches_done() - Cache clean interrupt received -+ * @kbdev: Kbase device -+ */ -+void kbase_clean_caches_done(struct kbase_device *kbdev); -+ -+/** -+ * kbase_instr_hwcnt_sample_done() - Dump complete interrupt received -+ * @kbdev: Kbase device -+ */ -+void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_INSTR_INTERNAL_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h -new file mode 100755 -index 000000000..8781561e7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h -@@ -0,0 +1,39 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Backend specific IRQ APIs -+ */ -+ -+#ifndef _KBASE_IRQ_INTERNAL_H_ -+#define _KBASE_IRQ_INTERNAL_H_ -+ -+int kbase_install_interrupts(struct kbase_device *kbdev); -+ -+void kbase_release_interrupts(struct kbase_device *kbdev); -+ -+/** -+ * kbase_synchronize_irqs - Ensure that all IRQ handlers have completed -+ * execution -+ * @kbdev: The kbase device -+ */ -+void kbase_synchronize_irqs(struct kbase_device *kbdev); -+ -+int kbasep_common_test_interrupt_handlers( -+ struct kbase_device * const kbdev); -+ -+#endif /* _KBASE_IRQ_INTERNAL_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c -new file mode 100755 -index 000000000..8416b80e8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c -@@ -0,0 +1,469 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+ -+#include -+ -+#if !defined(CONFIG_MALI_NO_MALI) -+ -+/* GPU IRQ Tags */ -+#define JOB_IRQ_TAG 0 -+#define MMU_IRQ_TAG 1 -+#define GPU_IRQ_TAG 2 -+ -+static void *kbase_tag(void *ptr, u32 tag) -+{ -+ return (void *)(((uintptr_t) ptr) | tag); -+} -+ -+static void *kbase_untag(void *ptr) -+{ -+ return (void *)(((uintptr_t) ptr) & ~3); -+} -+ -+static irqreturn_t kbase_job_irq_handler(int irq, void *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev = kbase_untag(data); -+ u32 val; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!kbdev->pm.backend.gpu_powered) { -+ /* GPU is turned off - IRQ is not for us */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ flags); -+ return IRQ_NONE; -+ } -+ -+ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); -+ -+#ifdef CONFIG_MALI_DEBUG -+ if (!kbdev->pm.backend.driver_ready_for_irqs) -+ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", -+ __func__, irq, val); -+#endif /* CONFIG_MALI_DEBUG */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!val) -+ return IRQ_NONE; -+ -+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -+ -+ kbase_job_done(kbdev, val); -+ -+ return IRQ_HANDLED; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_job_irq_handler); -+ -+static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev = kbase_untag(data); -+ u32 val; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!kbdev->pm.backend.gpu_powered) { -+ /* GPU is turned off - IRQ is not for us */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ flags); -+ return IRQ_NONE; -+ } -+ -+ atomic_inc(&kbdev->faults_pending); -+ -+ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); -+ -+#ifdef CONFIG_MALI_DEBUG -+ if (!kbdev->pm.backend.driver_ready_for_irqs) -+ dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", -+ __func__, irq, val); -+#endif /* CONFIG_MALI_DEBUG */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!val) { -+ atomic_dec(&kbdev->faults_pending); -+ return IRQ_NONE; -+ } -+ -+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -+ -+ kbase_mmu_interrupt(kbdev, val); -+ -+ atomic_dec(&kbdev->faults_pending); -+ -+ return IRQ_HANDLED; -+} -+ -+static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev = kbase_untag(data); -+ u32 val; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!kbdev->pm.backend.gpu_powered) { -+ /* GPU is turned off - IRQ is not for us */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ flags); -+ return IRQ_NONE; -+ } -+ -+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); -+ -+#ifdef CONFIG_MALI_DEBUG -+ if (!kbdev->pm.backend.driver_ready_for_irqs) -+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", -+ __func__, irq, val); -+#endif /* CONFIG_MALI_DEBUG */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!val) -+ return IRQ_NONE; -+ -+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -+ -+ kbase_gpu_interrupt(kbdev, val); -+ -+ return IRQ_HANDLED; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_gpu_irq_handler); -+ -+static irq_handler_t kbase_handler_table[] = { -+ [JOB_IRQ_TAG] = kbase_job_irq_handler, -+ [MMU_IRQ_TAG] = kbase_mmu_irq_handler, -+ [GPU_IRQ_TAG] = kbase_gpu_irq_handler, -+}; -+ -+#ifdef CONFIG_MALI_DEBUG -+#define JOB_IRQ_HANDLER JOB_IRQ_TAG -+#define MMU_IRQ_HANDLER MMU_IRQ_TAG -+#define GPU_IRQ_HANDLER GPU_IRQ_TAG -+ -+/** -+ * kbase_set_custom_irq_handler - Set a custom IRQ handler -+ * @kbdev: Device for which the handler is to be registered -+ * @custom_handler: Handler to be registered -+ * @irq_type: Interrupt type -+ * -+ * Registers given interrupt handler for requested interrupt type -+ * In the case where irq handler is not specified, the default handler shall be -+ * registered -+ * -+ * Return: 0 case success, error code otherwise -+ */ -+int kbase_set_custom_irq_handler(struct kbase_device *kbdev, -+ irq_handler_t custom_handler, -+ int irq_type) -+{ -+ int result = 0; -+ irq_handler_t requested_irq_handler = NULL; -+ -+ KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && -+ (GPU_IRQ_HANDLER >= irq_type)); -+ -+ /* Release previous handler */ -+ if (kbdev->irqs[irq_type].irq) -+ free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); -+ -+ requested_irq_handler = (NULL != custom_handler) ? custom_handler : -+ kbase_handler_table[irq_type]; -+ -+ if (0 != request_irq(kbdev->irqs[irq_type].irq, -+ requested_irq_handler, -+ kbdev->irqs[irq_type].flags | IRQF_SHARED, -+ dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { -+ result = -EINVAL; -+ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", -+ kbdev->irqs[irq_type].irq, irq_type); -+#ifdef CONFIG_SPARSE_IRQ -+ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); -+#endif /* CONFIG_SPARSE_IRQ */ -+ } -+ -+ return result; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); -+ -+/* test correct interrupt assigment and reception by cpu */ -+struct kbasep_irq_test { -+ struct hrtimer timer; -+ wait_queue_head_t wait; -+ int triggered; -+ u32 timeout; -+}; -+ -+static struct kbasep_irq_test kbasep_irq_test_data; -+ -+#define IRQ_TEST_TIMEOUT 500 -+ -+static irqreturn_t kbase_job_irq_test_handler(int irq, void *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev = kbase_untag(data); -+ u32 val; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!kbdev->pm.backend.gpu_powered) { -+ /* GPU is turned off - IRQ is not for us */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ flags); -+ return IRQ_NONE; -+ } -+ -+ val = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_STATUS), NULL); -+ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!val) -+ return IRQ_NONE; -+ -+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -+ -+ kbasep_irq_test_data.triggered = 1; -+ wake_up(&kbasep_irq_test_data.wait); -+ -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val, NULL); -+ -+ return IRQ_HANDLED; -+} -+ -+static irqreturn_t kbase_mmu_irq_test_handler(int irq, void *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev = kbase_untag(data); -+ u32 val; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!kbdev->pm.backend.gpu_powered) { -+ /* GPU is turned off - IRQ is not for us */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ flags); -+ return IRQ_NONE; -+ } -+ -+ val = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_STATUS), NULL); -+ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (!val) -+ return IRQ_NONE; -+ -+ dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -+ -+ kbasep_irq_test_data.triggered = 1; -+ wake_up(&kbasep_irq_test_data.wait); -+ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), val, NULL); -+ -+ return IRQ_HANDLED; -+} -+ -+static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) -+{ -+ struct kbasep_irq_test *test_data = container_of(timer, -+ struct kbasep_irq_test, timer); -+ -+ test_data->timeout = 1; -+ test_data->triggered = 1; -+ wake_up(&test_data->wait); -+ return HRTIMER_NORESTART; -+} -+ -+static int kbasep_common_test_interrupt( -+ struct kbase_device * const kbdev, u32 tag) -+{ -+ int err = 0; -+ irq_handler_t test_handler; -+ -+ u32 old_mask_val; -+ u16 mask_offset; -+ u16 rawstat_offset; -+ -+ switch (tag) { -+ case JOB_IRQ_TAG: -+ test_handler = kbase_job_irq_test_handler; -+ rawstat_offset = JOB_CONTROL_REG(JOB_IRQ_RAWSTAT); -+ mask_offset = JOB_CONTROL_REG(JOB_IRQ_MASK); -+ break; -+ case MMU_IRQ_TAG: -+ test_handler = kbase_mmu_irq_test_handler; -+ rawstat_offset = MMU_REG(MMU_IRQ_RAWSTAT); -+ mask_offset = MMU_REG(MMU_IRQ_MASK); -+ break; -+ case GPU_IRQ_TAG: -+ /* already tested by pm_driver - bail out */ -+ default: -+ return 0; -+ } -+ -+ /* store old mask */ -+ old_mask_val = kbase_reg_read(kbdev, mask_offset, NULL); -+ /* mask interrupts */ -+ kbase_reg_write(kbdev, mask_offset, 0x0, NULL); -+ -+ if (kbdev->irqs[tag].irq) { -+ /* release original handler and install test handler */ -+ if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { -+ err = -EINVAL; -+ } else { -+ kbasep_irq_test_data.timeout = 0; -+ hrtimer_init(&kbasep_irq_test_data.timer, -+ CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ kbasep_irq_test_data.timer.function = -+ kbasep_test_interrupt_timeout; -+ -+ /* trigger interrupt */ -+ kbase_reg_write(kbdev, mask_offset, 0x1, NULL); -+ kbase_reg_write(kbdev, rawstat_offset, 0x1, NULL); -+ -+ hrtimer_start(&kbasep_irq_test_data.timer, -+ HR_TIMER_DELAY_MSEC(IRQ_TEST_TIMEOUT), -+ HRTIMER_MODE_REL); -+ -+ wait_event(kbasep_irq_test_data.wait, -+ kbasep_irq_test_data.triggered != 0); -+ -+ if (kbasep_irq_test_data.timeout != 0) { -+ dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", -+ kbdev->irqs[tag].irq, tag); -+ err = -EINVAL; -+ } else { -+ dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", -+ kbdev->irqs[tag].irq, tag); -+ } -+ -+ hrtimer_cancel(&kbasep_irq_test_data.timer); -+ kbasep_irq_test_data.triggered = 0; -+ -+ /* mask interrupts */ -+ kbase_reg_write(kbdev, mask_offset, 0x0, NULL); -+ -+ /* release test handler */ -+ free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); -+ } -+ -+ /* restore original interrupt */ -+ if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], -+ kbdev->irqs[tag].flags | IRQF_SHARED, -+ dev_name(kbdev->dev), kbase_tag(kbdev, tag))) { -+ dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", -+ kbdev->irqs[tag].irq, tag); -+ err = -EINVAL; -+ } -+ } -+ /* restore old mask */ -+ kbase_reg_write(kbdev, mask_offset, old_mask_val, NULL); -+ -+ return err; -+} -+ -+int kbasep_common_test_interrupt_handlers( -+ struct kbase_device * const kbdev) -+{ -+ int err; -+ -+ init_waitqueue_head(&kbasep_irq_test_data.wait); -+ kbasep_irq_test_data.triggered = 0; -+ -+ /* A suspend won't happen during startup/insmod */ -+ kbase_pm_context_active(kbdev); -+ -+ err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); -+ if (err) { -+ dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); -+ goto out; -+ } -+ -+ err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); -+ if (err) { -+ dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); -+ goto out; -+ } -+ -+ dev_dbg(kbdev->dev, "Interrupts are correctly assigned.\n"); -+ -+ out: -+ kbase_pm_context_idle(kbdev); -+ -+ return err; -+} -+#endif /* CONFIG_MALI_DEBUG */ -+ -+int kbase_install_interrupts(struct kbase_device *kbdev) -+{ -+ u32 nr = ARRAY_SIZE(kbase_handler_table); -+ int err; -+ u32 i; -+ -+ for (i = 0; i < nr; i++) { -+ err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], -+ kbdev->irqs[i].flags | IRQF_SHARED, -+ dev_name(kbdev->dev), -+ kbase_tag(kbdev, i)); -+ if (err) { -+ dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", -+ kbdev->irqs[i].irq, i); -+#ifdef CONFIG_SPARSE_IRQ -+ dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); -+#endif /* CONFIG_SPARSE_IRQ */ -+ goto release; -+ } -+ } -+ -+ return 0; -+ -+ release: -+ while (i-- > 0) -+ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); -+ -+ return err; -+} -+ -+void kbase_release_interrupts(struct kbase_device *kbdev) -+{ -+ u32 nr = ARRAY_SIZE(kbase_handler_table); -+ u32 i; -+ -+ for (i = 0; i < nr; i++) { -+ if (kbdev->irqs[i].irq) -+ free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); -+ } -+} -+ -+void kbase_synchronize_irqs(struct kbase_device *kbdev) -+{ -+ u32 nr = ARRAY_SIZE(kbase_handler_table); -+ u32 i; -+ -+ for (i = 0; i < nr; i++) { -+ if (kbdev->irqs[i].irq) -+ synchronize_irq(kbdev->irqs[i].irq); -+ } -+} -+ -+#endif /* !defined(CONFIG_MALI_NO_MALI) */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c -new file mode 100755 -index 000000000..92358f2bf ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c -@@ -0,0 +1,237 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register backend context / address space management -+ */ -+ -+#include -+#include -+#include -+ -+/** -+ * assign_and_activate_kctx_addr_space - Assign an AS to a context -+ * @kbdev: Kbase device -+ * @kctx: Kbase context -+ * @current_as: Address Space to assign -+ * -+ * Assign an Address Space (AS) to a context, and add the context to the Policy. -+ * -+ * This includes -+ * setting up the global runpool_irq structure and the context on the AS, -+ * Activating the MMU on the AS, -+ * Allowing jobs to be submitted on the AS. -+ * -+ * Context: -+ * kbasep_js_kctx_info.jsctx_mutex held, -+ * kbasep_js_device_data.runpool_mutex held, -+ * AS transaction mutex held, -+ * Runpool IRQ lock held -+ */ -+static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ struct kbase_as *current_as) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* Attribute handling */ -+ kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); -+ -+ /* Allow it to run jobs */ -+ kbasep_js_set_submit_allowed(js_devdata, kctx); -+ -+ kbase_js_runpool_inc_context_count(kbdev, kctx); -+} -+ -+bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ int i; -+ -+ if (kbdev->hwaccess.active_kctx == kctx) { -+ /* Context is already active */ -+ return true; -+ } -+ -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { -+ if (kbdev->as_to_kctx[i] == kctx) { -+ /* Context already has ASID - mark as active */ -+ return true; -+ } -+ } -+ -+ /* Context does not have address space assigned */ -+ return false; -+} -+ -+void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ int as_nr = kctx->as_nr; -+ -+ if (as_nr == KBASEP_AS_NR_INVALID) { -+ WARN(1, "Attempting to release context without ASID\n"); -+ return; -+ } -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (atomic_read(&kctx->refcount) != 1) { -+ WARN(1, "Attempting to release active ASID\n"); -+ return; -+ } -+ -+ kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); -+ -+ kbase_ctx_sched_release_ctx(kctx); -+ kbase_js_runpool_dec_context_count(kbdev, kctx); -+} -+ -+void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+} -+ -+int kbase_backend_find_and_release_free_address_space( -+ struct kbase_device *kbdev, struct kbase_context *kctx) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ unsigned long flags; -+ int i; -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { -+ struct kbasep_js_kctx_info *as_js_kctx_info; -+ struct kbase_context *as_kctx; -+ -+ as_kctx = kbdev->as_to_kctx[i]; -+ as_js_kctx_info = &as_kctx->jctx.sched_info; -+ -+ /* Don't release privileged or active contexts, or contexts with -+ * jobs running. -+ * Note that a context will have at least 1 reference (which -+ * was previously taken by kbasep_js_schedule_ctx()) until -+ * descheduled. -+ */ -+ if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && -+ atomic_read(&as_kctx->refcount) == 1) { -+ if (!kbasep_js_runpool_retain_ctx_nolock(kbdev, -+ as_kctx)) { -+ WARN(1, "Failed to retain active context\n"); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, -+ flags); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ return KBASEP_AS_NR_INVALID; -+ } -+ -+ kbasep_js_clear_submit_allowed(js_devdata, as_kctx); -+ -+ /* Drop and retake locks to take the jsctx_mutex on the -+ * context we're about to release without violating lock -+ * ordering -+ */ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ -+ /* Release context from address space */ -+ mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ -+ kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); -+ -+ if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { -+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, -+ as_kctx, -+ true); -+ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); -+ -+ return i; -+ } -+ -+ /* Context was retained while locks were dropped, -+ * continue looking for free AS */ -+ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); -+ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ } -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ return KBASEP_AS_NR_INVALID; -+} -+ -+bool kbase_backend_use_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int as_nr) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbase_as *new_address_space = NULL; -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ if (kbdev->hwaccess.active_kctx == kctx) { -+ WARN(1, "Context is already scheduled in\n"); -+ return false; -+ } -+ -+ new_address_space = &kbdev->as[as_nr]; -+ -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ lockdep_assert_held(&kbdev->mmu_hw_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); -+ -+ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { -+ /* We need to retain it to keep the corresponding address space -+ */ -+ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); -+ } -+ -+ return true; -+} -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h -new file mode 100755 -index 000000000..08a7400e6 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h -@@ -0,0 +1,123 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register-based HW access backend specific definitions -+ */ -+ -+#ifndef _KBASE_HWACCESS_GPU_DEFS_H_ -+#define _KBASE_HWACCESS_GPU_DEFS_H_ -+ -+/* SLOT_RB_SIZE must be < 256 */ -+#define SLOT_RB_SIZE 2 -+#define SLOT_RB_MASK (SLOT_RB_SIZE - 1) -+ -+/** -+ * struct rb_entry - Ringbuffer entry -+ * @katom: Atom associated with this entry -+ */ -+struct rb_entry { -+ struct kbase_jd_atom *katom; -+}; -+ -+/** -+ * struct slot_rb - Slot ringbuffer -+ * @entries: Ringbuffer entries -+ * @last_context: The last context to submit a job on this slot -+ * @read_idx: Current read index of buffer -+ * @write_idx: Current write index of buffer -+ * @job_chain_flag: Flag used to implement jobchain disambiguation -+ */ -+struct slot_rb { -+ struct rb_entry entries[SLOT_RB_SIZE]; -+ -+ struct kbase_context *last_context; -+ -+ u8 read_idx; -+ u8 write_idx; -+ -+ u8 job_chain_flag; -+}; -+ -+/** -+ * struct kbase_backend_data - GPU backend specific data for HW access layer -+ * @slot_rb: Slot ringbuffers -+ * @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines -+ * whether slots 0/1 or slot 2 are currently being -+ * pulled from -+ * @scheduling_timer: The timer tick used for rescheduling jobs -+ * @timer_running: Is the timer running? The runpool_mutex must be -+ * held whilst modifying this. -+ * @suspend_timer: Is the timer suspended? Set when a suspend -+ * occurs and cleared on resume. The runpool_mutex -+ * must be held whilst modifying this. -+ * @reset_gpu: Set to a KBASE_RESET_xxx value (see comments) -+ * @reset_workq: Work queue for performing the reset -+ * @reset_work: Work item for performing the reset -+ * @reset_wait: Wait event signalled when the reset is complete -+ * @reset_timer: Timeout for soft-stops before the reset -+ * @timeouts_updated: Have timeout values just been updated? -+ * -+ * The hwaccess_lock (a spinlock) must be held when accessing this structure -+ */ -+struct kbase_backend_data { -+ struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; -+ -+ bool rmu_workaround_flag; -+ -+ struct hrtimer scheduling_timer; -+ -+ bool timer_running; -+ bool suspend_timer; -+ -+ atomic_t reset_gpu; -+ -+/* The GPU reset isn't pending */ -+#define KBASE_RESET_GPU_NOT_PENDING 0 -+/* kbase_prepare_to_reset_gpu has been called */ -+#define KBASE_RESET_GPU_PREPARED 1 -+/* kbase_reset_gpu has been called - the reset will now definitely happen -+ * within the timeout period */ -+#define KBASE_RESET_GPU_COMMITTED 2 -+/* The GPU reset process is currently occuring (timeout has expired or -+ * kbasep_try_reset_gpu_early was called) */ -+#define KBASE_RESET_GPU_HAPPENING 3 -+/* Reset the GPU silently, used when resetting the GPU as part of normal -+ * behavior (e.g. when exiting protected mode). */ -+#define KBASE_RESET_GPU_SILENT 4 -+ struct workqueue_struct *reset_workq; -+ struct work_struct reset_work; -+ wait_queue_head_t reset_wait; -+ struct hrtimer reset_timer; -+ -+ bool timeouts_updated; -+}; -+ -+/** -+ * struct kbase_jd_atom_backend - GPU backend specific katom data -+ */ -+struct kbase_jd_atom_backend { -+}; -+ -+/** -+ * struct kbase_context_backend - GPU backend specific context data -+ */ -+struct kbase_context_backend { -+}; -+ -+#endif /* _KBASE_HWACCESS_GPU_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c -new file mode 100755 -index 000000000..a6fb097b9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c -@@ -0,0 +1,1518 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Base kernel job manager APIs -+ */ -+ -+#include -+#include -+#include -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define beenthere(kctx, f, a...) \ -+ dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) -+ -+#if KBASE_GPU_RESET_EN -+static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev); -+static void kbasep_reset_timeout_worker(struct work_struct *data); -+static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer); -+#endif /* KBASE_GPU_RESET_EN */ -+ -+static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, -+ struct kbase_context *kctx) -+{ -+ return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx); -+} -+ -+void kbase_job_hw_submit(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, -+ int js) -+{ -+ struct kbase_context *kctx; -+ u32 cfg; -+ u64 jc_head = katom->jc; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ KBASE_DEBUG_ASSERT(katom); -+ -+ kctx = katom->kctx; -+ -+ /* Command register must be available */ -+ KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); -+ /* Affinity is not violating */ -+ kbase_js_debug_log_current_affinities(kbdev); -+ KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, -+ katom->affinity)); -+ -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), -+ jc_head & 0xFFFFFFFF, kctx); -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), -+ jc_head >> 32, kctx); -+ -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), -+ katom->affinity & 0xFFFFFFFF, kctx); -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI), -+ katom->affinity >> 32, kctx); -+ -+ /* start MMU, medium priority, cache clean/flush on end, clean/flush on -+ * start */ -+ cfg = kctx->as_nr; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) -+ cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; -+ -+#ifndef CONFIG_MALI_COH_GPU -+ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) -+ cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; -+ else -+ cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; -+ -+ if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END)) -+ cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; -+ else -+ cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; -+#endif /* CONFIG_MALI_COH_GPU */ -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649)) -+ cfg |= JS_CONFIG_START_MMU; -+ -+ cfg |= JS_CONFIG_THREAD_PRI(8); -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) && -+ (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED)) -+ cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; -+ -+ if (kbase_hw_has_feature(kbdev, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { -+ if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) { -+ cfg |= JS_CONFIG_JOB_CHAIN_FLAG; -+ katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; -+ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = -+ true; -+ } else { -+ katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; -+ kbdev->hwaccess.backend.slot_rb[js].job_chain_flag = -+ false; -+ } -+ } -+ -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx); -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT), -+ katom->flush_id, kctx); -+ -+ /* Write an approximate start timestamp. -+ * It's approximate because there might be a job in the HEAD register. -+ */ -+ katom->start_timestamp = ktime_get(); -+ -+ /* GO ! */ -+ dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", -+ katom, kctx, js, jc_head, katom->affinity); -+ -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, -+ (u32) katom->affinity); -+ -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_job_slots_event( -+ GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), -+ kctx, kbase_jd_atom_id(kctx, katom)); -+#endif -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(katom, jc_head, -+ katom->affinity, cfg); -+ KBASE_TLSTREAM_TL_RET_CTX_LPU( -+ kctx, -+ &kbdev->gpu_props.props.raw_props.js_features[ -+ katom->slot_nr]); -+ KBASE_TLSTREAM_TL_RET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); -+ KBASE_TLSTREAM_TL_RET_ATOM_LPU( -+ katom, -+ &kbdev->gpu_props.props.raw_props.js_features[js], -+ "ctx_nr,atom_nr"); -+#ifdef CONFIG_GPU_TRACEPOINTS -+ if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { -+ /* If this is the only job on the slot, trace it as starting */ -+ char js_string[16]; -+ -+ trace_gpu_sched_switch( -+ kbasep_make_job_slot_string(js, js_string, -+ sizeof(js_string)), -+ ktime_to_ns(katom->start_timestamp), -+ (u32)katom->kctx->id, 0, katom->work_id); -+ kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx; -+ } -+#endif -+ kbase_timeline_job_slot_submit(kbdev, kctx, katom, js); -+ -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), -+ JS_COMMAND_START, katom->kctx); -+} -+ -+/** -+ * kbasep_job_slot_update_head_start_timestamp - Update timestamp -+ * @kbdev: kbase device -+ * @js: job slot -+ * @end_timestamp: timestamp -+ * -+ * Update the start_timestamp of the job currently in the HEAD, based on the -+ * fact that we got an IRQ for the previous set of completed jobs. -+ * -+ * The estimate also takes into account the time the job was submitted, to -+ * work out the best estimate (which might still result in an over-estimate to -+ * the calculated time spent) -+ */ -+static void kbasep_job_slot_update_head_start_timestamp( -+ struct kbase_device *kbdev, -+ int js, -+ ktime_t end_timestamp) -+{ -+ if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) { -+ struct kbase_jd_atom *katom; -+ ktime_t timestamp_diff; -+ /* The atom in the HEAD */ -+ katom = kbase_gpu_inspect(kbdev, js, 0); -+ -+ KBASE_DEBUG_ASSERT(katom != NULL); -+ -+ timestamp_diff = ktime_sub(end_timestamp, -+ katom->start_timestamp); -+ if (ktime_to_ns(timestamp_diff) >= 0) { -+ /* Only update the timestamp if it's a better estimate -+ * than what's currently stored. This is because our -+ * estimate that accounts for the throttle time may be -+ * too much of an overestimate */ -+ katom->start_timestamp = end_timestamp; -+ } -+ } -+} -+ -+/** -+ * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline -+ * tracepoint -+ * @kbdev: kbase device -+ * @js: job slot -+ * -+ * Make a tracepoint call to the instrumentation module informing that -+ * softstop happened on given lpu (job slot). -+ */ -+static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, -+ int js) -+{ -+ KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP( -+ &kbdev->gpu_props.props.raw_props.js_features[js]); -+} -+ -+void kbase_job_done(struct kbase_device *kbdev, u32 done) -+{ -+ unsigned long flags; -+ int i; -+ u32 count = 0; -+ ktime_t end_timestamp = ktime_get(); -+ struct kbasep_js_device_data *js_devdata; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ js_devdata = &kbdev->js_data; -+ -+ KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done); -+ -+ memset(&kbdev->slot_submit_count_irq[0], 0, -+ sizeof(kbdev->slot_submit_count_irq)); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ while (done) { -+ u32 failed = done >> 16; -+ -+ /* treat failed slots as finished slots */ -+ u32 finished = (done & 0xFFFF) | failed; -+ -+ /* Note: This is inherently unfair, as we always check -+ * for lower numbered interrupts before the higher -+ * numbered ones.*/ -+ i = ffs(finished) - 1; -+ KBASE_DEBUG_ASSERT(i >= 0); -+ -+ do { -+ int nr_done; -+ u32 active; -+ u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */ -+ u64 job_tail = 0; -+ -+ if (failed & (1u << i)) { -+ /* read out the job slot status code if the job -+ * slot reported failure */ -+ completion_code = kbase_reg_read(kbdev, -+ JOB_SLOT_REG(i, JS_STATUS), NULL); -+ -+ switch (completion_code) { -+ case BASE_JD_EVENT_STOPPED: -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_job_slots_event( -+ GATOR_MAKE_EVENT( -+ GATOR_JOB_SLOT_SOFT_STOPPED, i), -+ NULL, 0); -+#endif -+ -+ kbasep_trace_tl_event_lpu_softstop( -+ kbdev, i); -+ -+ /* Soft-stopped job - read the value of -+ * JS_TAIL so that the job chain can -+ * be resumed */ -+ job_tail = (u64)kbase_reg_read(kbdev, -+ JOB_SLOT_REG(i, JS_TAIL_LO), -+ NULL) | -+ ((u64)kbase_reg_read(kbdev, -+ JOB_SLOT_REG(i, JS_TAIL_HI), -+ NULL) << 32); -+ break; -+ case BASE_JD_EVENT_NOT_STARTED: -+ /* PRLAM-10673 can cause a TERMINATED -+ * job to come back as NOT_STARTED, but -+ * the error interrupt helps us detect -+ * it */ -+ completion_code = -+ BASE_JD_EVENT_TERMINATED; -+ /* fall through */ -+ default: -+ dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", -+ i, completion_code, -+ kbase_exception_name -+ (kbdev, -+ completion_code)); -+ } -+ -+ kbase_gpu_irq_evict(kbdev, i); -+ } -+ -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), -+ done & ((1 << i) | (1 << (i + 16))), -+ NULL); -+ active = kbase_reg_read(kbdev, -+ JOB_CONTROL_REG(JOB_IRQ_JS_STATE), -+ NULL); -+ -+ if (((active >> i) & 1) == 0 && -+ (((done >> (i + 16)) & 1) == 0)) { -+ /* There is a potential race we must work -+ * around: -+ * -+ * 1. A job slot has a job in both current and -+ * next registers -+ * 2. The job in current completes -+ * successfully, the IRQ handler reads -+ * RAWSTAT and calls this function with the -+ * relevant bit set in "done" -+ * 3. The job in the next registers becomes the -+ * current job on the GPU -+ * 4. Sometime before the JOB_IRQ_CLEAR line -+ * above the job on the GPU _fails_ -+ * 5. The IRQ_CLEAR clears the done bit but not -+ * the failed bit. This atomically sets -+ * JOB_IRQ_JS_STATE. However since both jobs -+ * have now completed the relevant bits for -+ * the slot are set to 0. -+ * -+ * If we now did nothing then we'd incorrectly -+ * assume that _both_ jobs had completed -+ * successfully (since we haven't yet observed -+ * the fail bit being set in RAWSTAT). -+ * -+ * So at this point if there are no active jobs -+ * left we check to see if RAWSTAT has a failure -+ * bit set for the job slot. If it does we know -+ * that there has been a new failure that we -+ * didn't previously know about, so we make sure -+ * that we record this in active (but we wait -+ * for the next loop to deal with it). -+ * -+ * If we were handling a job failure (i.e. done -+ * has the relevant high bit set) then we know -+ * that the value read back from -+ * JOB_IRQ_JS_STATE is the correct number of -+ * remaining jobs because the failed job will -+ * have prevented any futher jobs from starting -+ * execution. -+ */ -+ u32 rawstat = kbase_reg_read(kbdev, -+ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); -+ -+ if ((rawstat >> (i + 16)) & 1) { -+ /* There is a failed job that we've -+ * missed - add it back to active */ -+ active |= (1u << i); -+ } -+ } -+ -+ dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", -+ completion_code); -+ -+ nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); -+ nr_done -= (active >> i) & 1; -+ nr_done -= (active >> (i + 16)) & 1; -+ -+ if (nr_done <= 0) { -+ dev_warn(kbdev->dev, "Spurious interrupt on slot %d", -+ i); -+ -+ goto spurious; -+ } -+ -+ count += nr_done; -+ -+ while (nr_done) { -+ if (nr_done == 1) { -+ kbase_gpu_complete_hw(kbdev, i, -+ completion_code, -+ job_tail, -+ &end_timestamp); -+ kbase_jm_try_kick_all(kbdev); -+ } else { -+ /* More than one job has completed. -+ * Since this is not the last job being -+ * reported this time it must have -+ * passed. This is because the hardware -+ * will not allow further jobs in a job -+ * slot to complete until the failed job -+ * is cleared from the IRQ status. -+ */ -+ kbase_gpu_complete_hw(kbdev, i, -+ BASE_JD_EVENT_DONE, -+ 0, -+ &end_timestamp); -+ } -+ nr_done--; -+ } -+ spurious: -+ done = kbase_reg_read(kbdev, -+ JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) { -+ /* Workaround for missing interrupt caused by -+ * PRLAM-10883 */ -+ if (((active >> i) & 1) && (0 == -+ kbase_reg_read(kbdev, -+ JOB_SLOT_REG(i, -+ JS_STATUS), NULL))) { -+ /* Force job slot to be processed again -+ */ -+ done |= (1u << i); -+ } -+ } -+ -+ failed = done >> 16; -+ finished = (done & 0xFFFF) | failed; -+ if (done) -+ end_timestamp = ktime_get(); -+ } while (finished & (1 << i)); -+ -+ kbasep_job_slot_update_head_start_timestamp(kbdev, i, -+ end_timestamp); -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+#if KBASE_GPU_RESET_EN -+ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == -+ KBASE_RESET_GPU_COMMITTED) { -+ /* If we're trying to reset the GPU then we might be able to do -+ * it early (without waiting for a timeout) because some jobs -+ * have completed -+ */ -+ kbasep_try_reset_gpu_early(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); -+} -+KBASE_EXPORT_TEST_API(kbase_job_done); -+ -+static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ bool soft_stops_allowed = true; -+ -+ if (kbase_jd_katom_is_protected(katom)) { -+ soft_stops_allowed = false; -+ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) { -+ if ((katom->core_req & BASE_JD_REQ_T) != 0) -+ soft_stops_allowed = false; -+ } -+ return soft_stops_allowed; -+} -+ -+static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, -+ base_jd_core_req core_reqs) -+{ -+ bool hard_stops_allowed = true; -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) { -+ if ((core_reqs & BASE_JD_REQ_T) != 0) -+ hard_stops_allowed = false; -+ } -+ return hard_stops_allowed; -+} -+ -+void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, -+ int js, -+ u32 action, -+ base_jd_core_req core_reqs, -+ struct kbase_jd_atom *target_katom) -+{ -+ struct kbase_context *kctx = target_katom->kctx; -+#if KBASE_TRACE_ENABLE -+ u32 status_reg_before; -+ u64 job_in_head_before; -+ u32 status_reg_after; -+ -+ KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK))); -+ -+ /* Check the head pointer */ -+ job_in_head_before = ((u64) kbase_reg_read(kbdev, -+ JOB_SLOT_REG(js, JS_HEAD_LO), NULL)) -+ | (((u64) kbase_reg_read(kbdev, -+ JOB_SLOT_REG(js, JS_HEAD_HI), NULL)) -+ << 32); -+ status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), -+ NULL); -+#endif -+ -+ if (action == JS_COMMAND_SOFT_STOP) { -+ bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, -+ target_katom); -+ -+ if (!soft_stop_allowed) { -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(kbdev->dev, -+ "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", -+ (unsigned int)core_reqs); -+#endif /* CONFIG_MALI_DEBUG */ -+ return; -+ } -+ -+ /* We are about to issue a soft stop, so mark the atom as having -+ * been soft stopped */ -+ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED; -+ -+ /* Mark the point where we issue the soft-stop command */ -+ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(target_katom); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { -+ int i; -+ -+ for (i = 0; -+ i < kbase_backend_nr_atoms_submitted(kbdev, js); -+ i++) { -+ struct kbase_jd_atom *katom; -+ -+ katom = kbase_gpu_inspect(kbdev, js, i); -+ -+ KBASE_DEBUG_ASSERT(katom); -+ -+ /* For HW_ISSUE_8316, only 'bad' jobs attacking -+ * the system can cause this issue: normally, -+ * all memory should be allocated in multiples -+ * of 4 pages, and growable memory should be -+ * changed size in multiples of 4 pages. -+ * -+ * Whilst such 'bad' jobs can be cleared by a -+ * GPU reset, the locking up of a uTLB entry -+ * caused by the bad job could also stall other -+ * ASs, meaning that other ASs' jobs don't -+ * complete in the 'grace' period before the -+ * reset. We don't want to lose other ASs' jobs -+ * when they would normally complete fine, so we -+ * must 'poke' the MMU regularly to help other -+ * ASs complete */ -+ kbase_as_poking_timer_retain_atom( -+ kbdev, katom->kctx, katom); -+ } -+ } -+ -+ if (kbase_hw_has_feature( -+ kbdev, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { -+ action = (target_katom->atom_flags & -+ KBASE_KATOM_FLAGS_JOBCHAIN) ? -+ JS_COMMAND_SOFT_STOP_1 : -+ JS_COMMAND_SOFT_STOP_0; -+ } -+ } else if (action == JS_COMMAND_HARD_STOP) { -+ bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, -+ core_reqs); -+ -+ if (!hard_stop_allowed) { -+ /* Jobs can be hard-stopped for the following reasons: -+ * * CFS decides the job has been running too long (and -+ * soft-stop has not occurred). In this case the GPU -+ * will be reset by CFS if the job remains on the -+ * GPU. -+ * -+ * * The context is destroyed, kbase_jd_zap_context -+ * will attempt to hard-stop the job. However it also -+ * has a watchdog which will cause the GPU to be -+ * reset if the job remains on the GPU. -+ * -+ * * An (unhandled) MMU fault occurred. As long as -+ * BASE_HW_ISSUE_8245 is defined then the GPU will be -+ * reset. -+ * -+ * All three cases result in the GPU being reset if the -+ * hard-stop fails, so it is safe to just return and -+ * ignore the hard-stop request. -+ */ -+ dev_warn(kbdev->dev, -+ "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", -+ (unsigned int)core_reqs); -+ return; -+ } -+ target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; -+ -+ if (kbase_hw_has_feature( -+ kbdev, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { -+ action = (target_katom->atom_flags & -+ KBASE_KATOM_FLAGS_JOBCHAIN) ? -+ JS_COMMAND_HARD_STOP_1 : -+ JS_COMMAND_HARD_STOP_0; -+ } -+ } -+ -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx); -+ -+#if KBASE_TRACE_ENABLE -+ status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS), -+ NULL); -+ if (status_reg_after == BASE_JD_EVENT_ACTIVE) { -+ struct kbase_jd_atom *head; -+ struct kbase_context *head_kctx; -+ -+ head = kbase_gpu_inspect(kbdev, js, 0); -+ head_kctx = head->kctx; -+ -+ if (status_reg_before == BASE_JD_EVENT_ACTIVE) -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, -+ head, job_in_head_before, js); -+ else -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, -+ 0, js); -+ -+ switch (action) { -+ case JS_COMMAND_SOFT_STOP: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, -+ head, head->jc, js); -+ break; -+ case JS_COMMAND_SOFT_STOP_0: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, -+ head, head->jc, js); -+ break; -+ case JS_COMMAND_SOFT_STOP_1: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, -+ head, head->jc, js); -+ break; -+ case JS_COMMAND_HARD_STOP: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, -+ head, head->jc, js); -+ break; -+ case JS_COMMAND_HARD_STOP_0: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, -+ head, head->jc, js); -+ break; -+ case JS_COMMAND_HARD_STOP_1: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, -+ head, head->jc, js); -+ break; -+ default: -+ BUG(); -+ break; -+ } -+ } else { -+ if (status_reg_before == BASE_JD_EVENT_ACTIVE) -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, -+ job_in_head_before, js); -+ else -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, -+ 0, js); -+ -+ switch (action) { -+ case JS_COMMAND_SOFT_STOP: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, -+ js); -+ break; -+ case JS_COMMAND_SOFT_STOP_0: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, -+ 0, js); -+ break; -+ case JS_COMMAND_SOFT_STOP_1: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, -+ 0, js); -+ break; -+ case JS_COMMAND_HARD_STOP: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, -+ js); -+ break; -+ case JS_COMMAND_HARD_STOP_0: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, -+ 0, js); -+ break; -+ case JS_COMMAND_HARD_STOP_1: -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, -+ 0, js); -+ break; -+ default: -+ BUG(); -+ break; -+ } -+ } -+#endif -+} -+ -+void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev; -+ struct kbasep_js_device_data *js_devdata; -+ int i; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ kbdev = kctx->kbdev; -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ js_devdata = &kbdev->js_data; -+ -+ /* Cancel any remaining running jobs for this kctx */ -+ mutex_lock(&kctx->jctx.lock); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* Invalidate all jobs in context, to prevent re-submitting */ -+ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { -+ if (!work_pending(&kctx->jctx.atoms[i].work)) -+ kctx->jctx.atoms[i].event_code = -+ BASE_JD_EVENT_JOB_CANCELLED; -+ } -+ -+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) -+ kbase_job_slot_hardstop(kctx, i, NULL); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kctx->jctx.lock); -+} -+ -+void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, -+ struct kbase_jd_atom *target_katom) -+{ -+ struct kbase_device *kbdev; -+ int js = target_katom->slot_nr; -+ int priority = target_katom->sched_priority; -+ int i; -+ bool stop_sent = false; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ kbdev = kctx->kbdev; -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { -+ struct kbase_jd_atom *katom; -+ -+ katom = kbase_gpu_inspect(kbdev, js, i); -+ if (!katom) -+ continue; -+ -+ if (katom->kctx != kctx) -+ continue; -+ -+ if (katom->sched_priority > priority) { -+ if (!stop_sent) -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE( -+ target_katom); -+ -+ kbase_job_slot_softstop(kbdev, js, katom); -+ stop_sent = true; -+ } -+ } -+} -+ -+struct zap_reset_data { -+ /* The stages are: -+ * 1. The timer has never been called -+ * 2. The zap has timed out, all slots are soft-stopped - the GPU reset -+ * will happen. The GPU has been reset when -+ * kbdev->hwaccess.backend.reset_waitq is signalled -+ * -+ * (-1 - The timer has been cancelled) -+ */ -+ int stage; -+ struct kbase_device *kbdev; -+ struct hrtimer timer; -+ spinlock_t lock; /* protects updates to stage member */ -+}; -+ -+static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer) -+{ -+ struct zap_reset_data *reset_data = container_of(timer, -+ struct zap_reset_data, timer); -+ struct kbase_device *kbdev = reset_data->kbdev; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&reset_data->lock, flags); -+ -+ if (reset_data->stage == -1) -+ goto out; -+ -+#if KBASE_GPU_RESET_EN -+ if (kbase_prepare_to_reset_gpu(kbdev)) { -+ dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", -+ ZAP_TIMEOUT); -+ kbase_reset_gpu(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ reset_data->stage = 2; -+ -+ out: -+ spin_unlock_irqrestore(&reset_data->lock, flags); -+ -+ return HRTIMER_NORESTART; -+} -+ -+void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct zap_reset_data reset_data; -+ unsigned long flags; -+ -+ hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC, -+ HRTIMER_MODE_REL); -+ reset_data.timer.function = zap_timeout_callback; -+ -+ spin_lock_init(&reset_data.lock); -+ -+ reset_data.kbdev = kbdev; -+ reset_data.stage = 1; -+ -+ hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT), -+ HRTIMER_MODE_REL); -+ -+ /* Wait for all jobs to finish, and for the context to be not-scheduled -+ * (due to kbase_job_zap_context(), we also guarentee it's not in the JS -+ * policy queue either */ -+ wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0); -+ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, -+ !kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ spin_lock_irqsave(&reset_data.lock, flags); -+ if (reset_data.stage == 1) { -+ /* The timer hasn't run yet - so cancel it */ -+ reset_data.stage = -1; -+ } -+ spin_unlock_irqrestore(&reset_data.lock, flags); -+ -+ hrtimer_cancel(&reset_data.timer); -+ -+ if (reset_data.stage == 2) { -+ /* The reset has already started. -+ * Wait for the reset to complete -+ */ -+ wait_event(kbdev->hwaccess.backend.reset_wait, -+ atomic_read(&kbdev->hwaccess.backend.reset_gpu) -+ == KBASE_RESET_GPU_NOT_PENDING); -+ } -+ destroy_hrtimer_on_stack(&reset_data.timer); -+ -+ dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); -+ -+ /* Ensure that the signallers of the waitqs have finished */ -+ mutex_lock(&kctx->jctx.lock); -+ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ mutex_unlock(&kctx->jctx.lock); -+} -+ -+u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) -+{ -+ u32 flush_id = 0; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { -+ mutex_lock(&kbdev->pm.lock); -+ if (kbdev->pm.backend.gpu_powered) -+ flush_id = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(LATEST_FLUSH), NULL); -+ mutex_unlock(&kbdev->pm.lock); -+ } -+ -+ return flush_id; -+} -+ -+int kbase_job_slot_init(struct kbase_device *kbdev) -+{ -+#if KBASE_GPU_RESET_EN -+ kbdev->hwaccess.backend.reset_workq = alloc_workqueue( -+ "Mali reset workqueue", 0, 1); -+ if (NULL == kbdev->hwaccess.backend.reset_workq) -+ return -EINVAL; -+ -+ KBASE_DEBUG_ASSERT(0 == -+ object_is_on_stack(&kbdev->hwaccess.backend.reset_work)); -+ INIT_WORK(&kbdev->hwaccess.backend.reset_work, -+ kbasep_reset_timeout_worker); -+ -+ hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, -+ HRTIMER_MODE_REL); -+ kbdev->hwaccess.backend.reset_timer.function = -+ kbasep_reset_timer_callback; -+#endif -+ -+ return 0; -+} -+KBASE_EXPORT_TEST_API(kbase_job_slot_init); -+ -+void kbase_job_slot_halt(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+void kbase_job_slot_term(struct kbase_device *kbdev) -+{ -+#if KBASE_GPU_RESET_EN -+ destroy_workqueue(kbdev->hwaccess.backend.reset_workq); -+#endif -+} -+KBASE_EXPORT_TEST_API(kbase_job_slot_term); -+ -+#if KBASE_GPU_RESET_EN -+/** -+ * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot -+ * @kbdev: kbase device pointer -+ * @kctx: context to check against -+ * @js: slot to check -+ * @target_katom: An atom to check, or NULL if all atoms from @kctx on -+ * slot @js should be checked -+ * -+ * This checks are based upon parameters that would normally be passed to -+ * kbase_job_slot_hardstop(). -+ * -+ * In the event of @target_katom being NULL, this will check the last jobs that -+ * are likely to be running on the slot to see if a) they belong to kctx, and -+ * so would be stopped, and b) whether they have AFBC -+ * -+ * In that case, It's guaranteed that a job currently executing on the HW with -+ * AFBC will be detected. However, this is a conservative check because it also -+ * detects jobs that have just completed too. -+ * -+ * Return: true when hard-stop _might_ stop an afbc atom, else false. -+ */ -+static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev, -+ struct kbase_context *kctx, int js, -+ struct kbase_jd_atom *target_katom) -+{ -+ bool ret = false; -+ int i; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* When we have an atom the decision can be made straight away. */ -+ if (target_katom) -+ return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC); -+ -+ /* Otherwise, we must chweck the hardware to see if it has atoms from -+ * this context with AFBC. */ -+ for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { -+ struct kbase_jd_atom *katom; -+ -+ katom = kbase_gpu_inspect(kbdev, js, i); -+ if (!katom) -+ continue; -+ -+ /* Ignore atoms from other contexts, they won't be stopped when -+ * we use this for checking if we should hard-stop them */ -+ if (katom->kctx != kctx) -+ continue; -+ -+ /* An atom on this slot and this context: check for AFBC */ -+ if (katom->core_req & BASE_JD_REQ_FS_AFBC) { -+ ret = true; -+ break; -+ } -+ } -+ -+ return ret; -+} -+#endif /* KBASE_GPU_RESET_EN */ -+ -+/** -+ * kbase_job_slot_softstop_swflags - Soft-stop a job with flags -+ * @kbdev: The kbase device -+ * @js: The job slot to soft-stop -+ * @target_katom: The job that should be soft-stopped (or NULL for any job) -+ * @sw_flags: Flags to pass in about the soft-stop -+ * -+ * Context: -+ * The job slot lock must be held when calling this function. -+ * The job slot must not already be in the process of being soft-stopped. -+ * -+ * Soft-stop the specified job slot, with extra information about the stop -+ * -+ * Where possible any job in the next register is evicted before the soft-stop. -+ */ -+void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, -+ struct kbase_jd_atom *target_katom, u32 sw_flags) -+{ -+ KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); -+ kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, -+ JS_COMMAND_SOFT_STOP | sw_flags); -+} -+ -+/** -+ * kbase_job_slot_softstop - Soft-stop the specified job slot -+ * @kbdev: The kbase device -+ * @js: The job slot to soft-stop -+ * @target_katom: The job that should be soft-stopped (or NULL for any job) -+ * Context: -+ * The job slot lock must be held when calling this function. -+ * The job slot must not already be in the process of being soft-stopped. -+ * -+ * Where possible any job in the next register is evicted before the soft-stop. -+ */ -+void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, -+ struct kbase_jd_atom *target_katom) -+{ -+ kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); -+} -+ -+/** -+ * kbase_job_slot_hardstop - Hard-stop the specified job slot -+ * @kctx: The kbase context that contains the job(s) that should -+ * be hard-stopped -+ * @js: The job slot to hard-stop -+ * @target_katom: The job that should be hard-stopped (or NULL for all -+ * jobs from the context) -+ * Context: -+ * The job slot lock must be held when calling this function. -+ */ -+void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, -+ struct kbase_jd_atom *target_katom) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ bool stopped; -+#if KBASE_GPU_RESET_EN -+ /* We make the check for AFBC before evicting/stopping atoms. Note -+ * that no other thread can modify the slots whilst we have the -+ * hwaccess_lock. */ -+ int needs_workaround_for_afbc = -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542) -+ && kbasep_check_for_afbc_on_slot(kbdev, kctx, js, -+ target_katom); -+#endif -+ -+ stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, -+ target_katom, -+ JS_COMMAND_HARD_STOP); -+#if KBASE_GPU_RESET_EN -+ if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) || -+ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) || -+ needs_workaround_for_afbc)) { -+ /* MIDBASE-2916 if a fragment job with AFBC encoding is -+ * hardstopped, ensure to do a soft reset also in order to -+ * clear the GPU status. -+ * Workaround for HW issue 8401 has an issue,so after -+ * hard-stopping just reset the GPU. This will ensure that the -+ * jobs leave the GPU.*/ -+ if (kbase_prepare_to_reset_gpu_locked(kbdev)) { -+ dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue"); -+ kbase_reset_gpu_locked(kbdev); -+ } -+ } -+#endif -+} -+ -+/** -+ * kbase_job_check_enter_disjoint - potentiall enter disjoint mode -+ * @kbdev: kbase device -+ * @action: the event which has occurred -+ * @core_reqs: core requirements of the atom -+ * @target_katom: the atom which is being affected -+ * -+ * For a certain soft/hard-stop action, work out whether to enter disjoint -+ * state. -+ * -+ * This does not register multiple disjoint events if the atom has already -+ * started a disjoint period -+ * -+ * @core_reqs can be supplied as 0 if the atom had not started on the hardware -+ * (and so a 'real' soft/hard-stop was not required, but it still interrupted -+ * flow, perhaps on another context) -+ * -+ * kbase_job_check_leave_disjoint() should be used to end the disjoint -+ * state when the soft/hard-stop action is complete -+ */ -+void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, -+ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) -+{ -+ u32 hw_action = action & JS_COMMAND_MASK; -+ -+ /* For hard-stop, don't enter if hard-stop not allowed */ -+ if (hw_action == JS_COMMAND_HARD_STOP && -+ !kbasep_hard_stop_allowed(kbdev, core_reqs)) -+ return; -+ -+ /* For soft-stop, don't enter if soft-stop not allowed, or isn't -+ * causing disjoint */ -+ if (hw_action == JS_COMMAND_SOFT_STOP && -+ !(kbasep_soft_stop_allowed(kbdev, target_katom) && -+ (action & JS_COMMAND_SW_CAUSES_DISJOINT))) -+ return; -+ -+ /* Nothing to do if already logged disjoint state on this atom */ -+ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) -+ return; -+ -+ target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; -+ kbase_disjoint_state_up(kbdev); -+} -+ -+/** -+ * kbase_job_check_enter_disjoint - potentially leave disjoint state -+ * @kbdev: kbase device -+ * @target_katom: atom which is finishing -+ * -+ * Work out whether to leave disjoint state when finishing an atom that was -+ * originated by kbase_job_check_enter_disjoint(). -+ */ -+void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, -+ struct kbase_jd_atom *target_katom) -+{ -+ if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { -+ target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; -+ kbase_disjoint_state_down(kbdev); -+ } -+} -+ -+ -+#if KBASE_GPU_RESET_EN -+static void kbase_debug_dump_registers(struct kbase_device *kbdev) -+{ -+ int i; -+ -+ kbase_io_history_dump(kbdev); -+ -+ dev_err(kbdev->dev, "Register state:"); -+ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL)); -+ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", -+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL), -+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL)); -+ for (i = 0; i < 3; i++) { -+ dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x", -+ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS), -+ NULL), -+ i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO), -+ NULL)); -+ } -+ dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", -+ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL)); -+ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL), -+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL), -+ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL)); -+ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL)); -+ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL)); -+ dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG), NULL)); -+} -+ -+static void kbasep_reset_timeout_worker(struct work_struct *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev; -+ ktime_t end_timestamp = ktime_get(); -+ struct kbasep_js_device_data *js_devdata; -+ bool try_schedule = false; -+ bool silent = false; -+ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; -+ -+ KBASE_DEBUG_ASSERT(data); -+ -+ kbdev = container_of(data, struct kbase_device, -+ hwaccess.backend.reset_work); -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ js_devdata = &kbdev->js_data; -+ -+ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == -+ KBASE_RESET_GPU_SILENT) -+ silent = true; -+ -+ KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); -+ -+ /* Suspend vinstr. -+ * This call will block until vinstr is suspended. */ -+ kbase_vinstr_suspend(kbdev->vinstr_ctx); -+ -+ /* Make sure the timer has completed - this cannot be done from -+ * interrupt context, so this cannot be done within -+ * kbasep_try_reset_gpu_early. */ -+ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); -+ -+ if (kbase_pm_context_active_handle_suspend(kbdev, -+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { -+ /* This would re-activate the GPU. Since it's already idle, -+ * there's no need to reset it */ -+ atomic_set(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_NOT_PENDING); -+ kbase_disjoint_state_down(kbdev); -+ wake_up(&kbdev->hwaccess.backend.reset_wait); -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ return; -+ } -+ -+ KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false); -+ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ spin_lock(&kbdev->hwaccess_lock); -+ spin_lock(&kbdev->mmu_mask_change); -+ /* We're about to flush out the IRQs and their bottom half's */ -+ kbdev->irq_reset_flush = true; -+ -+ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the -+ * spinlock; this also clears any outstanding interrupts */ -+ kbase_pm_disable_interrupts_nolock(kbdev); -+ -+ spin_unlock(&kbdev->mmu_mask_change); -+ spin_unlock(&kbdev->hwaccess_lock); -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ -+ /* Ensure that any IRQ handlers have finished -+ * Must be done without any locks IRQ handlers will take */ -+ kbase_synchronize_irqs(kbdev); -+ -+ /* Flush out any in-flight work items */ -+ kbase_flush_mmu_wqs(kbdev); -+ -+ /* The flush has completed so reset the active indicator */ -+ kbdev->irq_reset_flush = false; -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { -+ /* Ensure that L2 is not transitioning when we send the reset -+ * command */ -+ while (--max_loops && kbase_pm_get_trans_cores(kbdev, -+ KBASE_PM_CORE_L2)) -+ ; -+ -+ WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); -+ } -+ -+ mutex_lock(&kbdev->pm.lock); -+ /* We hold the pm lock, so there ought to be a current policy */ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy); -+ -+ /* All slot have been soft-stopped and we've waited -+ * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we -+ * assume that anything that is still left on the GPU is stuck there and -+ * we'll kill it when we reset the GPU */ -+ -+ if (!silent) -+ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", -+ RESET_TIMEOUT); -+ -+ /* Output the state of some interesting registers to help in the -+ * debugging of GPU resets */ -+ if (!silent) -+ kbase_debug_dump_registers(kbdev); -+ -+ /* Complete any jobs that were still on the GPU */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->protected_mode = false; -+ kbase_backend_reset(kbdev, &end_timestamp); -+ kbase_pm_metrics_update(kbdev, NULL); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* Reset the GPU */ -+ kbase_pm_init_hw(kbdev, 0); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_ctx_sched_restore_all_as(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ kbase_pm_enable_interrupts(kbdev); -+ -+ atomic_set(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_NOT_PENDING); -+ -+ kbase_disjoint_state_down(kbdev); -+ -+ wake_up(&kbdev->hwaccess.backend.reset_wait); -+ if (!silent) -+ dev_err(kbdev->dev, "Reset complete"); -+ -+ if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending) -+ try_schedule = true; -+ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ mutex_lock(&kbdev->pm.lock); -+ -+ /* Find out what cores are required now */ -+ kbase_pm_update_cores_state(kbdev); -+ -+ /* Synchronously request and wait for those cores, because if -+ * instrumentation is enabled it would need them immediately. */ -+ kbase_pm_check_transitions_sync(kbdev); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ -+ /* Try submitting some jobs to restart processing */ -+ if (try_schedule) { -+ KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, -+ 0); -+ kbase_js_sched_all(kbdev); -+ } -+ -+ /* Process any pending slot updates */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_backend_slot_update(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ kbase_pm_context_idle(kbdev); -+ -+ /* Release vinstr */ -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ -+ KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); -+} -+ -+static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) -+{ -+ struct kbase_device *kbdev = container_of(timer, struct kbase_device, -+ hwaccess.backend.reset_timer); -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ /* Reset still pending? */ -+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == -+ KBASE_RESET_GPU_COMMITTED) -+ queue_work(kbdev->hwaccess.backend.reset_workq, -+ &kbdev->hwaccess.backend.reset_work); -+ -+ return HRTIMER_NORESTART; -+} -+ -+/* -+ * If all jobs are evicted from the GPU then we can reset the GPU -+ * immediately instead of waiting for the timeout to elapse -+ */ -+ -+static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) -+{ -+ int i; -+ int pending_jobs = 0; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ /* Count the number of jobs */ -+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) -+ pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); -+ -+ if (pending_jobs > 0) { -+ /* There are still jobs on the GPU - wait */ -+ return; -+ } -+ -+ /* To prevent getting incorrect registers when dumping failed job, -+ * skip early reset. -+ */ -+ if (kbdev->job_fault_debug != false) -+ return; -+ -+ /* Check that the reset has been committed to (i.e. kbase_reset_gpu has -+ * been called), and that no other thread beat this thread to starting -+ * the reset */ -+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != -+ KBASE_RESET_GPU_COMMITTED) { -+ /* Reset has already occurred */ -+ return; -+ } -+ -+ queue_work(kbdev->hwaccess.backend.reset_workq, -+ &kbdev->hwaccess.backend.reset_work); -+} -+ -+static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ struct kbasep_js_device_data *js_devdata; -+ -+ js_devdata = &kbdev->js_data; -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbasep_try_reset_gpu_early_locked(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+/** -+ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU -+ * @kbdev: kbase device -+ * -+ * This function just soft-stops all the slots to ensure that as many jobs as -+ * possible are saved. -+ * -+ * Return: -+ * The function returns a boolean which should be interpreted as follows: -+ * true - Prepared for reset, kbase_reset_gpu_locked should be called. -+ * false - Another thread is performing a reset, kbase_reset_gpu should -+ * not be called. -+ */ -+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) -+{ -+ int i; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_NOT_PENDING, -+ KBASE_RESET_GPU_PREPARED) != -+ KBASE_RESET_GPU_NOT_PENDING) { -+ /* Some other thread is already resetting the GPU */ -+ return false; -+ } -+ -+ kbase_disjoint_state_up(kbdev); -+ -+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) -+ kbase_job_slot_softstop(kbdev, i, NULL); -+ -+ return true; -+} -+ -+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ bool ret; -+ struct kbasep_js_device_data *js_devdata; -+ -+ js_devdata = &kbdev->js_data; -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ ret = kbase_prepare_to_reset_gpu_locked(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return ret; -+} -+KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); -+ -+/* -+ * This function should be called after kbase_prepare_to_reset_gpu if it -+ * returns true. It should never be called without a corresponding call to -+ * kbase_prepare_to_reset_gpu. -+ * -+ * After this function is called (or not called if kbase_prepare_to_reset_gpu -+ * returned false), the caller should wait for -+ * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset -+ * has completed. -+ */ -+void kbase_reset_gpu(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ /* Note this is an assert/atomic_set because it is a software issue for -+ * a race to be occuring here */ -+ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == -+ KBASE_RESET_GPU_PREPARED); -+ atomic_set(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_COMMITTED); -+ -+ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", -+ kbdev->reset_timeout_ms); -+ -+ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, -+ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), -+ HRTIMER_MODE_REL); -+ -+ /* Try resetting early */ -+ kbasep_try_reset_gpu_early(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbase_reset_gpu); -+ -+void kbase_reset_gpu_locked(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ /* Note this is an assert/atomic_set because it is a software issue for -+ * a race to be occuring here */ -+ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == -+ KBASE_RESET_GPU_PREPARED); -+ atomic_set(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_COMMITTED); -+ -+ dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", -+ kbdev->reset_timeout_ms); -+ hrtimer_start(&kbdev->hwaccess.backend.reset_timer, -+ HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), -+ HRTIMER_MODE_REL); -+ -+ /* Try resetting early */ -+ kbasep_try_reset_gpu_early_locked(kbdev); -+} -+ -+void kbase_reset_gpu_silent(struct kbase_device *kbdev) -+{ -+ if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_NOT_PENDING, -+ KBASE_RESET_GPU_SILENT) != -+ KBASE_RESET_GPU_NOT_PENDING) { -+ /* Some other thread is already resetting the GPU */ -+ return; -+ } -+ -+ kbase_disjoint_state_up(kbdev); -+ -+ queue_work(kbdev->hwaccess.backend.reset_workq, -+ &kbdev->hwaccess.backend.reset_work); -+} -+ -+bool kbase_reset_gpu_active(struct kbase_device *kbdev) -+{ -+ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == -+ KBASE_RESET_GPU_NOT_PENDING) -+ return false; -+ -+ return true; -+} -+#endif /* KBASE_GPU_RESET_EN */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h -new file mode 100755 -index 000000000..1f382b3c1 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h -@@ -0,0 +1,164 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Job Manager backend-specific low-level APIs. -+ */ -+ -+#ifndef _KBASE_JM_HWACCESS_H_ -+#define _KBASE_JM_HWACCESS_H_ -+ -+#include -+#include -+#include -+ -+#include -+ -+/** -+ * kbase_job_submit_nolock() - Submit a job to a certain job-slot -+ * @kbdev: Device pointer -+ * @katom: Atom to submit -+ * @js: Job slot to submit on -+ * -+ * The caller must check kbasep_jm_is_submit_slots_free() != false before -+ * calling this. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must hold the hwaccess_lock -+ */ -+void kbase_job_submit_nolock(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, int js); -+ -+/** -+ * kbase_job_done_slot() - Complete the head job on a particular job-slot -+ * @kbdev: Device pointer -+ * @s: Job slot -+ * @completion_code: Completion code of job reported by GPU -+ * @job_tail: Job tail address reported by GPU -+ * @end_timestamp: Timestamp of job completion -+ */ -+void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, -+ u64 job_tail, ktime_t *end_timestamp); -+ -+#ifdef CONFIG_GPU_TRACEPOINTS -+static inline char *kbasep_make_job_slot_string(int js, char *js_string, -+ size_t js_size) -+{ -+ snprintf(js_string, js_size, "job_slot_%i", js); -+ return js_string; -+} -+#endif -+ -+/** -+ * kbase_job_hw_submit() - Submit a job to the GPU -+ * @kbdev: Device pointer -+ * @katom: Atom to submit -+ * @js: Job slot to submit on -+ * -+ * The caller must check kbasep_jm_is_submit_slots_free() != false before -+ * calling this. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must hold the hwaccess_lock -+ */ -+void kbase_job_hw_submit(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, -+ int js); -+ -+/** -+ * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop -+ * on the specified atom -+ * @kbdev: Device pointer -+ * @js: Job slot to stop on -+ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or -+ * JSn_COMMAND_SOFT_STOP -+ * @core_reqs: Core requirements of atom to stop -+ * @target_katom: Atom to stop -+ * -+ * The following locking conditions are made on the caller: -+ * - it must hold the hwaccess_lock -+ */ -+void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, -+ int js, -+ u32 action, -+ base_jd_core_req core_reqs, -+ struct kbase_jd_atom *target_katom); -+ -+/** -+ * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job -+ * slot belonging to a given context. -+ * @kbdev: Device pointer -+ * @kctx: Context pointer. May be NULL -+ * @katom: Specific atom to stop. May be NULL -+ * @js: Job slot to hard stop -+ * @action: The action to perform, either JSn_COMMAND_HARD_STOP or -+ * JSn_COMMAND_SOFT_STOP -+ * -+ * If no context is provided then all jobs on the slot will be soft or hard -+ * stopped. -+ * -+ * If a katom is provided then only that specific atom will be stopped. In this -+ * case the kctx parameter is ignored. -+ * -+ * Jobs that are on the slot but are not yet on the GPU will be unpulled and -+ * returned to the job scheduler. -+ * -+ * Return: true if an atom was stopped, false otherwise -+ */ -+bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js, -+ struct kbase_jd_atom *katom, -+ u32 action); -+ -+/** -+ * kbase_job_slot_init - Initialise job slot framework -+ * @kbdev: Device pointer -+ * -+ * Called on driver initialisation -+ * -+ * Return: 0 on success -+ */ -+int kbase_job_slot_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_job_slot_halt - Halt the job slot framework -+ * @kbdev: Device pointer -+ * -+ * Should prevent any further job slot processing -+ */ -+void kbase_job_slot_halt(struct kbase_device *kbdev); -+ -+/** -+ * kbase_job_slot_term - Terminate job slot framework -+ * @kbdev: Device pointer -+ * -+ * Called on driver termination -+ */ -+void kbase_job_slot_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpu_cacheclean - Cause a GPU cache clean & flush -+ * @kbdev: Device pointer -+ * -+ * Caller must not be in IRQ context -+ */ -+void kbase_gpu_cacheclean(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_JM_HWACCESS_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c -new file mode 100755 -index 000000000..4b4541660 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c -@@ -0,0 +1,1952 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register-based HW access backend specific APIs -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/* Return whether the specified ringbuffer is empty. HW access lock must be -+ * held */ -+#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) -+/* Return number of atoms currently in the specified ringbuffer. HW access lock -+ * must be held */ -+#define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) -+ -+static void kbase_gpu_release_atom(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, -+ ktime_t *end_timestamp); -+ -+/** -+ * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer -+ * @kbdev: Device pointer -+ * @katom: Atom to enqueue -+ * -+ * Context: Caller must hold the HW access lock -+ */ -+static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; -+ -+ WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; -+ rb->write_idx++; -+ -+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; -+} -+ -+/** -+ * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once -+ * it has been completed -+ * @kbdev: Device pointer -+ * @js: Job slot to remove atom from -+ * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in -+ * which case current time will be used. -+ * -+ * Context: Caller must hold the HW access lock -+ * -+ * Return: Atom removed from ringbuffer -+ */ -+static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, -+ int js, -+ ktime_t *end_timestamp) -+{ -+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; -+ struct kbase_jd_atom *katom; -+ -+ if (SLOT_RB_EMPTY(rb)) { -+ WARN(1, "GPU ringbuffer unexpectedly empty\n"); -+ return NULL; -+ } -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; -+ -+ kbase_gpu_release_atom(kbdev, katom, end_timestamp); -+ -+ rb->read_idx++; -+ -+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; -+ -+ kbase_js_debug_log_current_affinities(kbdev); -+ -+ return katom; -+} -+ -+struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, -+ int idx) -+{ -+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if ((SLOT_RB_ENTRIES(rb) - 1) < idx) -+ return NULL; /* idx out of range */ -+ -+ return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; -+} -+ -+struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, -+ int js) -+{ -+ return kbase_gpu_inspect(kbdev, js, 0); -+} -+ -+struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, -+ int js) -+{ -+ struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; -+ -+ if (SLOT_RB_EMPTY(rb)) -+ return NULL; -+ -+ return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; -+} -+ -+/** -+ * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently -+ * on the GPU -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * -+ * Return: true if there are atoms on the GPU for slot js, -+ * false otherwise -+ */ -+static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) -+{ -+ int i; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (i = 0; i < SLOT_RB_SIZE; i++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); -+ -+ if (!katom) -+ return false; -+ if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || -+ katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) -+ return true; -+ } -+ -+ return false; -+} -+ -+/** -+ * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms -+ * currently on the GPU -+ * @kbdev: Device pointer -+ * -+ * Return: true if there are any atoms on the GPU, false otherwise -+ */ -+static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) -+{ -+ int js; -+ int i; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ for (i = 0; i < SLOT_RB_SIZE; i++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); -+ -+ if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) -+ return true; -+ } -+ } -+ return false; -+} -+ -+int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) -+{ -+ int nr = 0; -+ int i; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (i = 0; i < SLOT_RB_SIZE; i++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); -+ -+ if (katom && (katom->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_SUBMITTED)) -+ nr++; -+ } -+ -+ return nr; -+} -+ -+int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) -+{ -+ int nr = 0; -+ int i; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (i = 0; i < SLOT_RB_SIZE; i++) { -+ if (kbase_gpu_inspect(kbdev, js, i)) -+ nr++; -+ } -+ -+ return nr; -+} -+ -+static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, -+ enum kbase_atom_gpu_rb_state min_rb_state) -+{ -+ int nr = 0; -+ int i; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (i = 0; i < SLOT_RB_SIZE; i++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); -+ -+ if (katom && (katom->gpu_rb_state >= min_rb_state)) -+ nr++; -+ } -+ -+ return nr; -+} -+ -+/** -+ * check_secure_atom - Check if the given atom is in the given secure state and -+ * has a ringbuffer state of at least -+ * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION -+ * @katom: Atom pointer -+ * @secure: Desired secure state -+ * -+ * Return: true if atom is in the given state, false otherwise -+ */ -+static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) -+{ -+ if (katom->gpu_rb_state >= -+ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && -+ ((kbase_jd_katom_is_protected(katom) && secure) || -+ (!kbase_jd_katom_is_protected(katom) && !secure))) -+ return true; -+ -+ return false; -+} -+ -+/** -+ * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given -+ * secure state in the ringbuffers of at least -+ * state -+ * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE -+ * @kbdev: Device pointer -+ * @secure: Desired secure state -+ * -+ * Return: true if any atoms are in the given state, false otherwise -+ */ -+static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, -+ bool secure) -+{ -+ int js, i; -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ for (i = 0; i < SLOT_RB_SIZE; i++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, -+ js, i); -+ -+ if (katom) { -+ if (check_secure_atom(katom, secure)) -+ return true; -+ } -+ } -+ } -+ -+ return false; -+} -+ -+int kbase_backend_slot_free(struct kbase_device *kbdev, int js) -+{ -+ if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != -+ KBASE_RESET_GPU_NOT_PENDING) { -+ /* The GPU is being reset - so prevent submission */ -+ return 0; -+ } -+ -+ return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); -+} -+ -+ -+static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, -+ int js, -+ struct kbase_jd_atom *katom) -+{ -+ /* The most recently checked affinity. Having this at this scope allows -+ * us to guarantee that we've checked the affinity in this function -+ * call. -+ */ -+ u64 recently_chosen_affinity = 0; -+ bool chosen_affinity = false; -+ bool retry; -+ -+ do { -+ retry = false; -+ -+ /* NOTE: The following uses a number of FALLTHROUGHs to optimize -+ * the calls to this function. Ending of the function is -+ * indicated by BREAK OUT */ -+ switch (katom->coreref_state) { -+ /* State when job is first attempted to be run */ -+ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: -+ KBASE_DEBUG_ASSERT(katom->affinity == 0); -+ -+ /* Compute affinity */ -+ if (false == kbase_js_choose_affinity( -+ &recently_chosen_affinity, kbdev, katom, -+ js)) { -+ /* No cores are currently available */ -+ /* *** BREAK OUT: No state transition *** */ -+ break; -+ } -+ -+ chosen_affinity = true; -+ -+ /* Request the cores */ -+ kbase_pm_request_cores(kbdev, -+ katom->core_req & BASE_JD_REQ_T, -+ recently_chosen_affinity); -+ -+ katom->affinity = recently_chosen_affinity; -+ -+ /* Proceed to next state */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; -+ -+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: -+ { -+ enum kbase_pm_cores_ready cores_ready; -+ -+ KBASE_DEBUG_ASSERT(katom->affinity != 0 || -+ (katom->core_req & BASE_JD_REQ_T)); -+ -+ cores_ready = kbase_pm_register_inuse_cores( -+ kbdev, -+ katom->core_req & BASE_JD_REQ_T, -+ katom->affinity); -+ if (cores_ready == KBASE_NEW_AFFINITY) { -+ /* Affinity no longer valid - return to -+ * previous state */ -+ kbasep_js_job_check_deref_cores(kbdev, -+ katom); -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, -+ JS_CORE_REF_REGISTER_INUSE_FAILED, -+ katom->kctx, katom, -+ katom->jc, js, -+ (u32) katom->affinity); -+ /* *** BREAK OUT: Return to previous -+ * state, retry *** */ -+ retry = true; -+ break; -+ } -+ if (cores_ready == KBASE_CORES_NOT_READY) { -+ /* Stay in this state and return, to -+ * retry at this state later */ -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, -+ JS_CORE_REF_REGISTER_INUSE_FAILED, -+ katom->kctx, katom, -+ katom->jc, js, -+ (u32) katom->affinity); -+ /* *** BREAK OUT: No state transition -+ * *** */ -+ break; -+ } -+ /* Proceed to next state */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; -+ } -+ -+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: -+ KBASE_DEBUG_ASSERT(katom->affinity != 0 || -+ (katom->core_req & BASE_JD_REQ_T)); -+ -+ /* Optimize out choosing the affinity twice in the same -+ * function call */ -+ if (chosen_affinity == false) { -+ /* See if the affinity changed since a previous -+ * call. */ -+ if (false == kbase_js_choose_affinity( -+ &recently_chosen_affinity, -+ kbdev, katom, js)) { -+ /* No cores are currently available */ -+ kbasep_js_job_check_deref_cores(kbdev, -+ katom); -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, -+ JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, -+ katom->kctx, katom, -+ katom->jc, js, -+ (u32) recently_chosen_affinity); -+ /* *** BREAK OUT: Transition to lower -+ * state *** */ -+ break; -+ } -+ chosen_affinity = true; -+ } -+ -+ /* Now see if this requires a different set of cores */ -+ if (recently_chosen_affinity != katom->affinity) { -+ enum kbase_pm_cores_ready cores_ready; -+ -+ kbase_pm_request_cores(kbdev, -+ katom->core_req & BASE_JD_REQ_T, -+ recently_chosen_affinity); -+ -+ /* Register new cores whilst we still hold the -+ * old ones, to minimize power transitions */ -+ cores_ready = -+ kbase_pm_register_inuse_cores(kbdev, -+ katom->core_req & BASE_JD_REQ_T, -+ recently_chosen_affinity); -+ kbasep_js_job_check_deref_cores(kbdev, katom); -+ -+ /* Fixup the state that was reduced by -+ * deref_cores: */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; -+ katom->affinity = recently_chosen_affinity; -+ if (cores_ready == KBASE_NEW_AFFINITY) { -+ /* Affinity no longer valid - return to -+ * previous state */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; -+ -+ kbasep_js_job_check_deref_cores(kbdev, -+ katom); -+ -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, -+ JS_CORE_REF_REGISTER_INUSE_FAILED, -+ katom->kctx, katom, -+ katom->jc, js, -+ (u32) katom->affinity); -+ /* *** BREAK OUT: Return to previous -+ * state, retry *** */ -+ retry = true; -+ break; -+ } -+ /* Now might be waiting for powerup again, with -+ * a new affinity */ -+ if (cores_ready == KBASE_CORES_NOT_READY) { -+ /* Return to previous state */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, -+ JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, -+ katom->kctx, katom, -+ katom->jc, js, -+ (u32) katom->affinity); -+ /* *** BREAK OUT: Transition to lower -+ * state *** */ -+ break; -+ } -+ } -+ /* Proceed to next state */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; -+ -+ /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: -+ KBASE_DEBUG_ASSERT(katom->affinity != 0 || -+ (katom->core_req & BASE_JD_REQ_T)); -+ KBASE_DEBUG_ASSERT(katom->affinity == -+ recently_chosen_affinity); -+ -+ /* Note: this is where the caller must've taken the -+ * hwaccess_lock */ -+ -+ /* Check for affinity violations - if there are any, -+ * then we just ask the caller to requeue and try again -+ * later */ -+ if (kbase_js_affinity_would_violate(kbdev, js, -+ katom->affinity) != false) { -+ /* Return to previous state */ -+ katom->coreref_state = -+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; -+ /* *** BREAK OUT: Transition to lower state *** -+ */ -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, -+ JS_CORE_REF_AFFINITY_WOULD_VIOLATE, -+ katom->kctx, katom, katom->jc, js, -+ (u32) katom->affinity); -+ break; -+ } -+ -+ /* No affinity violations would result, so the cores are -+ * ready */ -+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; -+ /* *** BREAK OUT: Cores Ready *** */ -+ break; -+ -+ default: -+ KBASE_DEBUG_ASSERT_MSG(false, -+ "Unhandled kbase_atom_coreref_state %d", -+ katom->coreref_state); -+ break; -+ } -+ } while (retry != false); -+ -+ return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); -+} -+ -+static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(katom != NULL); -+ -+ switch (katom->coreref_state) { -+ case KBASE_ATOM_COREREF_STATE_READY: -+ /* State where atom was submitted to the HW - just proceed to -+ * power-down */ -+ KBASE_DEBUG_ASSERT(katom->affinity != 0 || -+ (katom->core_req & BASE_JD_REQ_T)); -+ -+ /* fallthrough */ -+ -+ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: -+ /* State where cores were registered */ -+ KBASE_DEBUG_ASSERT(katom->affinity != 0 || -+ (katom->core_req & BASE_JD_REQ_T)); -+ kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, -+ katom->affinity); -+ -+ break; -+ -+ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: -+ /* State where cores were requested, but not registered */ -+ KBASE_DEBUG_ASSERT(katom->affinity != 0 || -+ (katom->core_req & BASE_JD_REQ_T)); -+ kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, -+ katom->affinity); -+ break; -+ -+ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: -+ /* Initial state - nothing required */ -+ KBASE_DEBUG_ASSERT(katom->affinity == 0); -+ break; -+ -+ default: -+ KBASE_DEBUG_ASSERT_MSG(false, -+ "Unhandled coreref_state: %d", -+ katom->coreref_state); -+ break; -+ } -+ -+ katom->affinity = 0; -+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; -+} -+ -+static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, -+ base_jd_core_req core_req, u64 affinity, -+ enum kbase_atom_coreref_state coreref_state) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ switch (coreref_state) { -+ case KBASE_ATOM_COREREF_STATE_READY: -+ /* State where atom was submitted to the HW - just proceed to -+ * power-down */ -+ KBASE_DEBUG_ASSERT(affinity != 0 || -+ (core_req & BASE_JD_REQ_T)); -+ -+ /* fallthrough */ -+ -+ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: -+ /* State where cores were registered */ -+ KBASE_DEBUG_ASSERT(affinity != 0 || -+ (core_req & BASE_JD_REQ_T)); -+ kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, -+ affinity); -+ -+ break; -+ -+ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: -+ /* State where cores were requested, but not registered */ -+ KBASE_DEBUG_ASSERT(affinity != 0 || -+ (core_req & BASE_JD_REQ_T)); -+ kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, -+ affinity); -+ break; -+ -+ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: -+ /* Initial state - nothing required */ -+ KBASE_DEBUG_ASSERT(affinity == 0); -+ break; -+ -+ default: -+ KBASE_DEBUG_ASSERT_MSG(false, -+ "Unhandled coreref_state: %d", -+ coreref_state); -+ break; -+ } -+} -+ -+static void kbase_gpu_release_atom(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, -+ ktime_t *end_timestamp) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ switch (katom->gpu_rb_state) { -+ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: -+ /* Should be impossible */ -+ WARN(1, "Attempting to release atom not in ringbuffer\n"); -+ break; -+ -+ case KBASE_ATOM_GPU_RB_SUBMITTED: -+ /* Inform power management at start/finish of atom so it can -+ * update its GPU utilisation metrics. Mark atom as not -+ * submitted beforehand. */ -+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; -+ kbase_pm_metrics_update(kbdev, end_timestamp); -+ -+ if (katom->core_req & BASE_JD_REQ_PERMON) -+ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ -+ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, -+ &kbdev->gpu_props.props.raw_props.js_features -+ [katom->slot_nr]); -+ KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); -+ KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx, -+ &kbdev->gpu_props.props.raw_props.js_features -+ [katom->slot_nr]); -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_READY: -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: -+ kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, -+ katom->affinity); -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: -+ break; -+ -+ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: -+ if (katom->protected_state.enter != -+ KBASE_ATOM_ENTER_PROTECTED_CHECK || -+ katom->protected_state.exit != -+ KBASE_ATOM_EXIT_PROTECTED_CHECK) -+ kbdev->protected_mode_transition = false; -+ -+ if (kbase_jd_katom_is_protected(katom) && -+ (katom->protected_state.enter == -+ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ -+ /* Go back to configured model for IPA */ -+ kbase_ipa_model_use_configured_locked(kbdev); -+ } -+ -+ -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: -+ break; -+ } -+ -+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; -+ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; -+} -+ -+static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ kbase_gpu_release_atom(kbdev, katom, NULL); -+ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; -+} -+ -+static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ bool slot_busy[3]; -+ -+ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) -+ return true; -+ slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, -+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); -+ slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, -+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); -+ slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, -+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY); -+ -+ if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || -+ (js != 2 && !slot_busy[2])) -+ return true; -+ -+ /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ -+ if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || -+ kbase_gpu_atoms_submitted(kbdev, 1) || -+ backend->rmu_workaround_flag)) -+ return false; -+ -+ /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ -+ if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || -+ !backend->rmu_workaround_flag)) -+ return false; -+ -+ backend->rmu_workaround_flag = !backend->rmu_workaround_flag; -+ -+ return true; -+} -+ -+/** -+ * other_slots_busy - Determine if any job slots other than @js are currently -+ * running atoms -+ * @kbdev: Device pointer -+ * @js: Job slot -+ * -+ * Return: true if any slots other than @js are busy, false otherwise -+ */ -+static inline bool other_slots_busy(struct kbase_device *kbdev, int js) -+{ -+ int slot; -+ -+ for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { -+ if (slot == js) -+ continue; -+ -+ if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, -+ KBASE_ATOM_GPU_RB_SUBMITTED)) -+ return true; -+ } -+ -+ return false; -+} -+ -+static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) -+{ -+ return kbdev->protected_mode; -+} -+ -+static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) -+{ -+ int err = -EINVAL; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ONCE(!kbdev->protected_ops, -+ "Cannot enter protected mode: protected callbacks not specified.\n"); -+ -+ /* -+ * When entering into protected mode, we must ensure that the -+ * GPU is not operating in coherent mode as well. This is to -+ * ensure that no protected memory can be leaked. -+ */ -+ if (kbdev->system_coherency == COHERENCY_ACE) -+ kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); -+ -+ if (kbdev->protected_ops) { -+ /* Switch GPU to protected mode */ -+ err = kbdev->protected_ops->protected_mode_enable( -+ kbdev->protected_dev); -+ -+ if (err) -+ dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", -+ err); -+ else -+ kbdev->protected_mode = true; -+ } -+ -+ return err; -+} -+ -+static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ONCE(!kbdev->protected_ops, -+ "Cannot exit protected mode: protected callbacks not specified.\n"); -+ -+ if (!kbdev->protected_ops) -+ return -EINVAL; -+ -+ /* The protected mode disable callback will be called as part of reset -+ */ -+ kbase_reset_gpu_silent(kbdev); -+ -+ return 0; -+} -+ -+static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, -+ struct kbase_jd_atom **katom, int idx, int js) -+{ -+ int err = 0; -+ -+ switch (katom[idx]->protected_state.enter) { -+ case KBASE_ATOM_ENTER_PROTECTED_CHECK: -+ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev); -+ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV -+ * should ensure that we are not already transitiong, and that -+ * there are no atoms currently on the GPU. */ -+ WARN_ON(kbdev->protected_mode_transition); -+ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); -+ -+ kbdev->protected_mode_transition = true; -+ katom[idx]->protected_state.enter = -+ KBASE_ATOM_ENTER_PROTECTED_VINSTR; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_ENTER_PROTECTED_VINSTR: -+ if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { -+ /* -+ * We can't switch now because -+ * the vinstr core state switch -+ * is not done yet. -+ */ -+ return -EAGAIN; -+ } -+ -+ /* Use generic model for IPA in protected mode */ -+ kbase_ipa_model_use_fallback_locked(kbdev); -+ -+ /* Once reaching this point GPU must be -+ * switched to protected mode or vinstr -+ * re-enabled. */ -+ -+ /* -+ * Not in correct mode, begin protected mode switch. -+ * Entering protected mode requires us to power down the L2, -+ * and drop out of fully coherent mode. -+ */ -+ katom[idx]->protected_state.enter = -+ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: -+ /* Avoid unnecessary waiting on non-ACE platforms. */ -+ if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { -+ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || -+ kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { -+ /* -+ * The L2 is still powered, wait for all the users to -+ * finish with it before doing the actual reset. -+ */ -+ return -EAGAIN; -+ } -+ } -+ -+ katom[idx]->protected_state.enter = -+ KBASE_ATOM_ENTER_PROTECTED_FINISHED; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_ENTER_PROTECTED_FINISHED: -+ -+ /* No jobs running, so we can switch GPU mode right now. */ -+ err = kbase_gpu_protected_mode_enter(kbdev); -+ -+ /* -+ * Regardless of result, we are no longer transitioning -+ * the GPU. -+ */ -+ kbdev->protected_mode_transition = false; -+ KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev); -+ if (err) { -+ /* -+ * Failed to switch into protected mode, resume -+ * vinstr core and fail atom. -+ */ -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; -+ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); -+ /* Only return if head atom or previous atom -+ * already removed - as atoms must be returned -+ * in order. */ -+ if (idx == 0 || katom[0]->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ kbase_jm_return_atom_to_js(kbdev, katom[idx]); -+ } -+ -+ /* Go back to configured model for IPA */ -+ kbase_ipa_model_use_configured_locked(kbdev); -+ -+ return -EINVAL; -+ } -+ -+ /* Protected mode sanity checks. */ -+ KBASE_DEBUG_ASSERT_MSG( -+ kbase_jd_katom_is_protected(katom[idx]) == -+ kbase_gpu_in_protected_mode(kbdev), -+ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", -+ kbase_jd_katom_is_protected(katom[idx]), -+ kbase_gpu_in_protected_mode(kbdev)); -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_READY; -+ } -+ -+ return 0; -+} -+ -+static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, -+ struct kbase_jd_atom **katom, int idx, int js) -+{ -+ int err = 0; -+ -+ -+ switch (katom[idx]->protected_state.exit) { -+ case KBASE_ATOM_EXIT_PROTECTED_CHECK: -+ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev); -+ /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV -+ * should ensure that we are not already transitiong, and that -+ * there are no atoms currently on the GPU. */ -+ WARN_ON(kbdev->protected_mode_transition); -+ WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); -+ -+ /* -+ * Exiting protected mode requires a reset, but first the L2 -+ * needs to be powered down to ensure it's not active when the -+ * reset is issued. -+ */ -+ katom[idx]->protected_state.exit = -+ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; -+ -+ kbdev->protected_mode_transition = true; -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: -+ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || -+ kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { -+ /* -+ * The L2 is still powered, wait for all the users to -+ * finish with it before doing the actual reset. -+ */ -+ return -EAGAIN; -+ } -+ katom[idx]->protected_state.exit = -+ KBASE_ATOM_EXIT_PROTECTED_RESET; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_EXIT_PROTECTED_RESET: -+ /* Issue the reset to the GPU */ -+ err = kbase_gpu_protected_mode_reset(kbdev); -+ -+ if (err) { -+ kbdev->protected_mode_transition = false; -+ -+ /* Failed to exit protected mode, fail atom */ -+ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; -+ kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); -+ /* Only return if head atom or previous atom -+ * already removed - as atoms must be returned -+ * in order */ -+ if (idx == 0 || katom[0]->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ kbase_jm_return_atom_to_js(kbdev, katom[idx]); -+ } -+ -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ -+ /* Use generic model for IPA in protected mode */ -+ kbase_ipa_model_use_fallback_locked(kbdev); -+ -+ return -EINVAL; -+ } -+ -+ katom[idx]->protected_state.exit = -+ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: -+ /* A GPU reset is issued when exiting protected mode. Once the -+ * reset is done all atoms' state will also be reset. For this -+ * reason, if the atom is still in this state we can safely -+ * say that the reset has not completed i.e., we have not -+ * finished exiting protected mode yet. -+ */ -+ return -EAGAIN; -+ } -+ -+ return 0; -+} -+ -+void kbase_backend_slot_update(struct kbase_device *kbdev) -+{ -+ int js; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ struct kbase_jd_atom *katom[2]; -+ int idx; -+ -+ katom[0] = kbase_gpu_inspect(kbdev, js, 0); -+ katom[1] = kbase_gpu_inspect(kbdev, js, 1); -+ WARN_ON(katom[1] && !katom[0]); -+ -+ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { -+ bool cores_ready; -+ int ret; -+ -+ if (!katom[idx]) -+ continue; -+ -+ switch (katom[idx]->gpu_rb_state) { -+ case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: -+ /* Should be impossible */ -+ WARN(1, "Attempting to update atom not in ringbuffer\n"); -+ break; -+ -+ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: -+ if (katom[idx]->atom_flags & -+ KBASE_KATOM_FLAG_X_DEP_BLOCKED) -+ break; -+ -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: -+ if (kbase_gpu_check_secure_atoms(kbdev, -+ !kbase_jd_katom_is_protected( -+ katom[idx]))) -+ break; -+ -+ if ((idx == 1) && (kbase_jd_katom_is_protected( -+ katom[0]) != -+ kbase_jd_katom_is_protected( -+ katom[1]))) -+ break; -+ -+ if (kbdev->protected_mode_transition) -+ break; -+ -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: -+ -+ /* -+ * Exiting protected mode must be done before -+ * the references on the cores are taken as -+ * a power down the L2 is required which -+ * can't happen after the references for this -+ * atom are taken. -+ */ -+ -+ if (!kbase_gpu_in_protected_mode(kbdev) && -+ kbase_jd_katom_is_protected(katom[idx])) { -+ /* Atom needs to transition into protected mode. */ -+ ret = kbase_jm_enter_protected_mode(kbdev, -+ katom, idx, js); -+ if (ret) -+ break; -+ } else if (kbase_gpu_in_protected_mode(kbdev) && -+ !kbase_jd_katom_is_protected(katom[idx])) { -+ /* Atom needs to transition out of protected mode. */ -+ ret = kbase_jm_exit_protected_mode(kbdev, -+ katom, idx, js); -+ if (ret) -+ break; -+ } -+ katom[idx]->protected_state.exit = -+ KBASE_ATOM_EXIT_PROTECTED_CHECK; -+ -+ /* Atom needs no protected mode transition. */ -+ -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: -+ if (katom[idx]->will_fail_event_code) { -+ kbase_gpu_mark_atom_for_return(kbdev, -+ katom[idx]); -+ /* Set EVENT_DONE so this atom will be -+ completed, not unpulled. */ -+ katom[idx]->event_code = -+ BASE_JD_EVENT_DONE; -+ /* Only return if head atom or previous -+ * atom already removed - as atoms must -+ * be returned in order. */ -+ if (idx == 0 || katom[0]->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ kbase_jm_return_atom_to_js(kbdev, katom[idx]); -+ } -+ break; -+ } -+ -+ cores_ready = -+ kbasep_js_job_check_ref_cores(kbdev, js, -+ katom[idx]); -+ -+ if (katom[idx]->event_code == -+ BASE_JD_EVENT_PM_EVENT) { -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_RETURN_TO_JS; -+ break; -+ } -+ -+ if (!cores_ready) -+ break; -+ -+ kbase_js_affinity_retain_slot_cores(kbdev, js, -+ katom[idx]->affinity); -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: -+ if (!kbase_gpu_rmu_workaround(kbdev, js)) -+ break; -+ -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_READY; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_READY: -+ -+ if (idx == 1) { -+ /* Only submit if head atom or previous -+ * atom already submitted */ -+ if ((katom[0]->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_SUBMITTED && -+ katom[0]->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) -+ break; -+ -+ /* If intra-slot serialization in use -+ * then don't submit atom to NEXT slot -+ */ -+ if (kbdev->serialize_jobs & -+ KBASE_SERIALIZE_INTRA_SLOT) -+ break; -+ } -+ -+ /* If inter-slot serialization in use then don't -+ * submit atom if any other slots are in use */ -+ if ((kbdev->serialize_jobs & -+ KBASE_SERIALIZE_INTER_SLOT) && -+ other_slots_busy(kbdev, js)) -+ break; -+ -+ if ((kbdev->serialize_jobs & -+ KBASE_SERIALIZE_RESET) && -+ kbase_reset_gpu_active(kbdev)) -+ break; -+ -+ /* Check if this job needs the cycle counter -+ * enabled before submission */ -+ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) -+ kbase_pm_request_gpu_cycle_counter_l2_is_on( -+ kbdev); -+ -+ kbase_job_hw_submit(kbdev, katom[idx], js); -+ katom[idx]->gpu_rb_state = -+ KBASE_ATOM_GPU_RB_SUBMITTED; -+ -+ /* Inform power management at start/finish of -+ * atom so it can update its GPU utilisation -+ * metrics. */ -+ kbase_pm_metrics_update(kbdev, -+ &katom[idx]->start_timestamp); -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_SUBMITTED: -+ /* Atom submitted to HW, nothing else to do */ -+ break; -+ -+ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: -+ /* Only return if head atom or previous atom -+ * already removed - as atoms must be returned -+ * in order */ -+ if (idx == 0 || katom[0]->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ kbase_jm_return_atom_to_js(kbdev, -+ katom[idx]); -+ } -+ break; -+ } -+ } -+ } -+ -+ /* Warn if PRLAM-8987 affinity restrictions are violated */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) -+ WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || -+ kbase_gpu_atoms_submitted(kbdev, 1)) && -+ kbase_gpu_atoms_submitted(kbdev, 2)); -+} -+ -+ -+void kbase_backend_run_atom(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ kbase_gpu_enqueue_atom(kbdev, katom); -+ kbase_backend_slot_update(kbdev); -+} -+ -+#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ -+ (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) -+ -+bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) -+{ -+ struct kbase_jd_atom *katom; -+ struct kbase_jd_atom *next_katom; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ katom = kbase_gpu_inspect(kbdev, js, 0); -+ next_katom = kbase_gpu_inspect(kbdev, js, 1); -+ -+ if (next_katom && katom->kctx == next_katom->kctx && -+ next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && -+ HAS_DEP(next_katom) && -+ (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) -+ != 0 || -+ kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) -+ != 0)) { -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), -+ JS_COMMAND_NOP, NULL); -+ next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; -+ -+ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, -+ &kbdev->gpu_props.props.raw_props.js_features -+ [katom->slot_nr]); -+ KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as -+ [katom->kctx->as_nr]); -+ KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx, -+ &kbdev->gpu_props.props.raw_props.js_features -+ [katom->slot_nr]); -+ -+ return true; -+ } -+ -+ return false; -+} -+ -+void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, -+ u32 completion_code, -+ u64 job_tail, -+ ktime_t *end_timestamp) -+{ -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); -+ struct kbase_context *kctx = katom->kctx; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* -+ * When a hard-stop is followed close after a soft-stop, the completion -+ * code may be set to STOPPED, even though the job is terminated -+ */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { -+ if (completion_code == BASE_JD_EVENT_STOPPED && -+ (katom->atom_flags & -+ KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { -+ completion_code = BASE_JD_EVENT_TERMINATED; -+ } -+ } -+ -+ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & -+ BASE_JD_REQ_SKIP_CACHE_END)) && -+ completion_code != BASE_JD_EVENT_DONE && -+ !(completion_code & BASE_JD_SW_EVENT)) { -+ /* When a job chain fails, on a T60x or when -+ * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not -+ * flushed. To prevent future evictions causing possible memory -+ * corruption we need to flush the cache manually before any -+ * affected memory gets reused. */ -+ katom->need_cache_flush_cores_retained = katom->affinity; -+ kbase_pm_request_cores(kbdev, false, katom->affinity); -+ } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { -+ if (kbdev->gpu_props.num_core_groups > 1 && -+ !(katom->affinity & -+ kbdev->gpu_props.props.coherency_info.group[0].core_mask -+ ) && -+ (katom->affinity & -+ kbdev->gpu_props.props.coherency_info.group[1].core_mask -+ )) { -+ dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); -+ katom->need_cache_flush_cores_retained = -+ katom->affinity; -+ kbase_pm_request_cores(kbdev, false, -+ katom->affinity); -+ } -+ } -+ -+ katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); -+ kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); -+ -+ if (completion_code == BASE_JD_EVENT_STOPPED) { -+ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, -+ 0); -+ -+ /* -+ * Dequeue next atom from ringbuffers on same slot if required. -+ * This atom will already have been removed from the NEXT -+ * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that -+ * the atoms on this slot are returned in the correct order. -+ */ -+ if (next_katom && katom->kctx == next_katom->kctx && -+ next_katom->sched_priority == -+ katom->sched_priority) { -+ kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); -+ kbase_jm_return_atom_to_js(kbdev, next_katom); -+ } -+ } else if (completion_code != BASE_JD_EVENT_DONE) { -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ int i; -+ -+#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 -+ KBASE_TRACE_DUMP(kbdev); -+#endif -+ kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); -+ -+ /* -+ * Remove all atoms on the same context from ringbuffers. This -+ * will not remove atoms that are already on the GPU, as these -+ * are guaranteed not to have fail dependencies on the failed -+ * atom. -+ */ -+ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { -+ struct kbase_jd_atom *katom_idx0 = -+ kbase_gpu_inspect(kbdev, i, 0); -+ struct kbase_jd_atom *katom_idx1 = -+ kbase_gpu_inspect(kbdev, i, 1); -+ -+ if (katom_idx0 && katom_idx0->kctx == katom->kctx && -+ HAS_DEP(katom_idx0) && -+ katom_idx0->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_SUBMITTED) { -+ /* Dequeue katom_idx0 from ringbuffer */ -+ kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); -+ -+ if (katom_idx1 && -+ katom_idx1->kctx == katom->kctx -+ && HAS_DEP(katom_idx1) && -+ katom_idx0->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_SUBMITTED) { -+ /* Dequeue katom_idx1 from ringbuffer */ -+ kbase_gpu_dequeue_atom(kbdev, i, -+ end_timestamp); -+ -+ katom_idx1->event_code = -+ BASE_JD_EVENT_STOPPED; -+ kbase_jm_return_atom_to_js(kbdev, -+ katom_idx1); -+ } -+ katom_idx0->event_code = BASE_JD_EVENT_STOPPED; -+ kbase_jm_return_atom_to_js(kbdev, katom_idx0); -+ -+ } else if (katom_idx1 && -+ katom_idx1->kctx == katom->kctx && -+ HAS_DEP(katom_idx1) && -+ katom_idx1->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_SUBMITTED) { -+ /* Can not dequeue this atom yet - will be -+ * dequeued when atom at idx0 completes */ -+ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; -+ kbase_gpu_mark_atom_for_return(kbdev, -+ katom_idx1); -+ } -+ } -+ } -+ -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, -+ js, completion_code); -+ -+ if (job_tail != 0 && job_tail != katom->jc) { -+ bool was_updated = (job_tail != katom->jc); -+ -+ /* Some of the job has been executed, so we update the job chain -+ * address to where we should resume from */ -+ katom->jc = job_tail; -+ if (was_updated) -+ KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, -+ katom, job_tail, js); -+ } -+ -+ /* Only update the event code for jobs that weren't cancelled */ -+ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) -+ katom->event_code = (base_jd_event_code)completion_code; -+ -+ kbase_device_trace_register_access(kctx, REG_WRITE, -+ JOB_CONTROL_REG(JOB_IRQ_CLEAR), -+ 1 << js); -+ -+ /* Complete the job, and start new ones -+ * -+ * Also defer remaining work onto the workqueue: -+ * - Re-queue Soft-stopped jobs -+ * - For any other jobs, queue the job back into the dependency system -+ * - Schedule out the parent context if necessary, and schedule a new -+ * one in. -+ */ -+#ifdef CONFIG_GPU_TRACEPOINTS -+ { -+ /* The atom in the HEAD */ -+ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, -+ 0); -+ -+ if (next_katom && next_katom->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_SUBMITTED) { -+ char js_string[16]; -+ -+ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, -+ js_string, -+ sizeof(js_string)), -+ ktime_to_ns(*end_timestamp), -+ (u32)next_katom->kctx->id, 0, -+ next_katom->work_id); -+ kbdev->hwaccess.backend.slot_rb[js].last_context = -+ next_katom->kctx; -+ } else { -+ char js_string[16]; -+ -+ trace_gpu_sched_switch(kbasep_make_job_slot_string(js, -+ js_string, -+ sizeof(js_string)), -+ ktime_to_ns(ktime_get()), 0, 0, -+ 0); -+ kbdev->hwaccess.backend.slot_rb[js].last_context = 0; -+ } -+ } -+#endif -+ -+ if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) -+ kbase_reset_gpu_silent(kbdev); -+ -+ if (completion_code == BASE_JD_EVENT_STOPPED) -+ katom = kbase_jm_return_atom_to_js(kbdev, katom); -+ else -+ katom = kbase_jm_complete(kbdev, katom, end_timestamp); -+ -+ if (katom) { -+ /* Cross-slot dependency has now become runnable. Try to submit -+ * it. */ -+ -+ /* Check if there are lower priority jobs to soft stop */ -+ kbase_job_slot_ctx_priority_check_locked(kctx, katom); -+ -+ kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); -+ } -+ -+ /* Job completion may have unblocked other atoms. Try to update all job -+ * slots */ -+ kbase_backend_slot_update(kbdev); -+} -+ -+void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) -+{ -+ int js; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* Reset should always take the GPU out of protected mode */ -+ WARN_ON(kbase_gpu_in_protected_mode(kbdev)); -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ int atom_idx = 0; -+ int idx; -+ -+ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, -+ js, atom_idx); -+ bool keep_in_jm_rb = false; -+ -+ if (!katom) -+ break; -+ if (katom->protected_state.exit == -+ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) -+ { -+ KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); -+ -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ -+ /* protected mode sanity checks */ -+ KBASE_DEBUG_ASSERT_MSG( -+ kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), -+ "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", -+ kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); -+ KBASE_DEBUG_ASSERT_MSG( -+ (kbase_jd_katom_is_protected(katom) && js == 0) || -+ !kbase_jd_katom_is_protected(katom), -+ "Protected atom on JS%d not supported", js); -+ } -+ if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) -+ keep_in_jm_rb = true; -+ -+ kbase_gpu_release_atom(kbdev, katom, NULL); -+ -+ /* -+ * If the atom wasn't on HW when the reset was issued -+ * then leave it in the RB and next time we're kicked -+ * it will be processed again from the starting state. -+ */ -+ if (keep_in_jm_rb) { -+ kbasep_js_job_check_deref_cores(kbdev, katom); -+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; -+ katom->affinity = 0; -+ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; -+ /* As the atom was not removed, increment the -+ * index so that we read the correct atom in the -+ * next iteration. */ -+ atom_idx++; -+ continue; -+ } -+ -+ /* -+ * The atom was on the HW when the reset was issued -+ * all we can do is fail the atom. -+ */ -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ kbase_jm_complete(kbdev, katom, end_timestamp); -+ } -+ } -+ -+ kbdev->protected_mode_transition = false; -+} -+ -+static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, -+ int js, -+ struct kbase_jd_atom *katom, -+ u32 action) -+{ -+ u32 hw_action = action & JS_COMMAND_MASK; -+ -+ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); -+ kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, -+ katom->core_req, katom); -+ katom->kctx->blocked_js[js][katom->sched_priority] = true; -+} -+ -+static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, -+ u32 action, -+ bool disjoint) -+{ -+ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; -+ kbase_gpu_mark_atom_for_return(kbdev, katom); -+ katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; -+ -+ if (disjoint) -+ kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, -+ katom); -+} -+ -+static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) -+{ -+ if (katom->x_post_dep) { -+ struct kbase_jd_atom *dep_atom = katom->x_post_dep; -+ -+ if (dep_atom->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && -+ dep_atom->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_RETURN_TO_JS) -+ return dep_atom->slot_nr; -+ } -+ return -1; -+} -+ -+static void kbase_job_evicted(struct kbase_jd_atom *katom) -+{ -+ kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, -+ katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); -+} -+ -+bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js, -+ struct kbase_jd_atom *katom, -+ u32 action) -+{ -+ struct kbase_jd_atom *katom_idx0; -+ struct kbase_jd_atom *katom_idx1; -+ -+ bool katom_idx0_valid, katom_idx1_valid; -+ -+ bool ret = false; -+ -+ int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; -+ int prio_idx0 = 0, prio_idx1 = 0; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); -+ katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); -+ -+ if (katom_idx0) -+ prio_idx0 = katom_idx0->sched_priority; -+ if (katom_idx1) -+ prio_idx1 = katom_idx1->sched_priority; -+ -+ if (katom) { -+ katom_idx0_valid = (katom_idx0 == katom); -+ /* If idx0 is to be removed and idx1 is on the same context, -+ * then idx1 must also be removed otherwise the atoms might be -+ * returned out of order */ -+ if (katom_idx1) -+ katom_idx1_valid = (katom_idx1 == katom) || -+ (katom_idx0_valid && -+ (katom_idx0->kctx == -+ katom_idx1->kctx)); -+ else -+ katom_idx1_valid = false; -+ } else { -+ katom_idx0_valid = (katom_idx0 && -+ (!kctx || katom_idx0->kctx == kctx)); -+ katom_idx1_valid = (katom_idx1 && -+ (!kctx || katom_idx1->kctx == kctx) && -+ prio_idx0 == prio_idx1); -+ } -+ -+ if (katom_idx0_valid) -+ stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); -+ if (katom_idx1_valid) -+ stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); -+ -+ if (katom_idx0_valid) { -+ if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { -+ /* Simple case - just dequeue and return */ -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ if (katom_idx1_valid) { -+ kbase_gpu_dequeue_atom(kbdev, js, NULL); -+ katom_idx1->event_code = -+ BASE_JD_EVENT_REMOVED_FROM_NEXT; -+ kbase_jm_return_atom_to_js(kbdev, katom_idx1); -+ katom_idx1->kctx->blocked_js[js][prio_idx1] = -+ true; -+ } -+ -+ katom_idx0->event_code = -+ BASE_JD_EVENT_REMOVED_FROM_NEXT; -+ kbase_jm_return_atom_to_js(kbdev, katom_idx0); -+ katom_idx0->kctx->blocked_js[js][prio_idx0] = true; -+ } else { -+ /* katom_idx0 is on GPU */ -+ if (katom_idx1 && katom_idx1->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_SUBMITTED) { -+ /* katom_idx0 and katom_idx1 are on GPU */ -+ -+ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, -+ JS_COMMAND_NEXT), NULL) == 0) { -+ /* idx0 has already completed - stop -+ * idx1 if needed*/ -+ if (katom_idx1_valid) { -+ kbase_gpu_stop_atom(kbdev, js, -+ katom_idx1, -+ action); -+ ret = true; -+ } -+ } else { -+ /* idx1 is in NEXT registers - attempt -+ * to remove */ -+ kbase_reg_write(kbdev, -+ JOB_SLOT_REG(js, -+ JS_COMMAND_NEXT), -+ JS_COMMAND_NOP, NULL); -+ -+ if (kbase_reg_read(kbdev, -+ JOB_SLOT_REG(js, -+ JS_HEAD_NEXT_LO), NULL) -+ != 0 || -+ kbase_reg_read(kbdev, -+ JOB_SLOT_REG(js, -+ JS_HEAD_NEXT_HI), NULL) -+ != 0) { -+ /* idx1 removed successfully, -+ * will be handled in IRQ */ -+ kbase_job_evicted(katom_idx1); -+ kbase_gpu_remove_atom(kbdev, -+ katom_idx1, -+ action, true); -+ stop_x_dep_idx1 = -+ should_stop_x_dep_slot(katom_idx1); -+ -+ /* stop idx0 if still on GPU */ -+ kbase_gpu_stop_atom(kbdev, js, -+ katom_idx0, -+ action); -+ ret = true; -+ } else if (katom_idx1_valid) { -+ /* idx0 has already completed, -+ * stop idx1 if needed */ -+ kbase_gpu_stop_atom(kbdev, js, -+ katom_idx1, -+ action); -+ ret = true; -+ } -+ } -+ } else if (katom_idx1_valid) { -+ /* idx1 not on GPU but must be dequeued*/ -+ -+ /* idx1 will be handled in IRQ */ -+ kbase_gpu_remove_atom(kbdev, katom_idx1, action, -+ false); -+ /* stop idx0 */ -+ /* This will be repeated for anything removed -+ * from the next registers, since their normal -+ * flow was also interrupted, and this function -+ * might not enter disjoint state e.g. if we -+ * don't actually do a hard stop on the head -+ * atom */ -+ kbase_gpu_stop_atom(kbdev, js, katom_idx0, -+ action); -+ ret = true; -+ } else { -+ /* no atom in idx1 */ -+ /* just stop idx0 */ -+ kbase_gpu_stop_atom(kbdev, js, katom_idx0, -+ action); -+ ret = true; -+ } -+ } -+ } else if (katom_idx1_valid) { -+ if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { -+ /* Mark for return */ -+ /* idx1 will be returned once idx0 completes */ -+ kbase_gpu_remove_atom(kbdev, katom_idx1, action, -+ false); -+ } else { -+ /* idx1 is on GPU */ -+ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, -+ JS_COMMAND_NEXT), NULL) == 0) { -+ /* idx0 has already completed - stop idx1 */ -+ kbase_gpu_stop_atom(kbdev, js, katom_idx1, -+ action); -+ ret = true; -+ } else { -+ /* idx1 is in NEXT registers - attempt to -+ * remove */ -+ kbase_reg_write(kbdev, JOB_SLOT_REG(js, -+ JS_COMMAND_NEXT), -+ JS_COMMAND_NOP, NULL); -+ -+ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, -+ JS_HEAD_NEXT_LO), NULL) != 0 || -+ kbase_reg_read(kbdev, JOB_SLOT_REG(js, -+ JS_HEAD_NEXT_HI), NULL) != 0) { -+ /* idx1 removed successfully, will be -+ * handled in IRQ once idx0 completes */ -+ kbase_job_evicted(katom_idx1); -+ kbase_gpu_remove_atom(kbdev, katom_idx1, -+ action, -+ false); -+ } else { -+ /* idx0 has already completed - stop -+ * idx1 */ -+ kbase_gpu_stop_atom(kbdev, js, -+ katom_idx1, -+ action); -+ ret = true; -+ } -+ } -+ } -+ } -+ -+ -+ if (stop_x_dep_idx0 != -1) -+ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, -+ NULL, action); -+ -+ if (stop_x_dep_idx1 != -1) -+ kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, -+ NULL, action); -+ -+ return ret; -+} -+ -+void kbase_gpu_cacheclean(struct kbase_device *kbdev) -+{ -+ /* Limit the number of loops to avoid a hang if the interrupt is missed -+ */ -+ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; -+ -+ mutex_lock(&kbdev->cacheclean_lock); -+ -+ /* use GPU_COMMAND completion solution */ -+ /* clean & invalidate the caches */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAN_INV_CACHES, NULL); -+ -+ /* wait for cache flush to complete before continuing */ -+ while (--max_loops && -+ (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & -+ CLEAN_CACHES_COMPLETED) == 0) -+ ; -+ -+ /* clear the CLEAN_CACHES_COMPLETED irq */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, -+ CLEAN_CACHES_COMPLETED); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), -+ CLEAN_CACHES_COMPLETED, NULL); -+ KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != -+ KBASE_INSTR_STATE_CLEANING, -+ "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); -+ -+ mutex_unlock(&kbdev->cacheclean_lock); -+} -+ -+void kbase_backend_cacheclean(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ if (katom->need_cache_flush_cores_retained) { -+ unsigned long flags; -+ -+ kbase_gpu_cacheclean(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_pm_unrequest_cores(kbdev, false, -+ katom->need_cache_flush_cores_retained); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ katom->need_cache_flush_cores_retained = 0; -+ } -+} -+ -+void kbase_backend_complete_wq(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ /* -+ * If cache flush required due to HW workaround then perform the flush -+ * now -+ */ -+ kbase_backend_cacheclean(kbdev, katom); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && -+ (katom->core_req & BASE_JD_REQ_FS) && -+ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && -+ (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && -+ !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { -+ dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); -+ if (kbasep_10969_workaround_clamp_coordinates(katom)) { -+ /* The job had a TILE_RANGE_FAULT after was soft-stopped -+ * Due to an HW issue we try to execute the job again. -+ */ -+ dev_dbg(kbdev->dev, -+ "Clamping has been executed, try to rerun the job\n" -+ ); -+ katom->event_code = BASE_JD_EVENT_STOPPED; -+ katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; -+ } -+ } -+ -+ /* Clear the coreref_state now - while check_deref_cores() may not have -+ * been called yet, the caller will have taken a copy of this field. If -+ * this is not done, then if the atom is re-scheduled (following a soft -+ * stop) then the core reference would not be retaken. */ -+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; -+ katom->affinity = 0; -+} -+ -+void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, -+ base_jd_core_req core_req, u64 affinity, -+ enum kbase_atom_coreref_state coreref_state) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, -+ coreref_state); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (!kbdev->pm.active_count) { -+ mutex_lock(&kbdev->js_data.runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ kbase_pm_update_active(kbdev); -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&kbdev->js_data.runpool_mutex); -+ } -+} -+ -+void kbase_gpu_dump_slots(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ unsigned long flags; -+ int js; -+ -+ js_devdata = &kbdev->js_data; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ int idx; -+ -+ for (idx = 0; idx < SLOT_RB_SIZE; idx++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, -+ js, -+ idx); -+ -+ if (katom) -+ dev_info(kbdev->dev, -+ " js%d idx%d : katom=%p gpu_rb_state=%d\n", -+ js, idx, katom, katom->gpu_rb_state); -+ else -+ dev_info(kbdev->dev, " js%d idx%d : empty\n", -+ js, idx); -+ } -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+ -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h -new file mode 100755 -index 000000000..1e0e05ad3 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h -@@ -0,0 +1,76 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register-based HW access backend specific APIs -+ */ -+ -+#ifndef _KBASE_HWACCESS_GPU_H_ -+#define _KBASE_HWACCESS_GPU_H_ -+ -+#include -+ -+/** -+ * kbase_gpu_irq_evict - Evict an atom from a NEXT slot -+ * -+ * @kbdev: Device pointer -+ * @js: Job slot to evict from -+ * -+ * Evict the atom in the NEXT slot for the specified job slot. This function is -+ * called from the job complete IRQ handler when the previous job has failed. -+ * -+ * Return: true if job evicted from NEXT registers, false otherwise -+ */ -+bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js); -+ -+/** -+ * kbase_gpu_complete_hw - Complete an atom on job slot js -+ * -+ * @kbdev: Device pointer -+ * @js: Job slot that has completed -+ * @completion_code: Event code from job that has completed -+ * @job_tail: The tail address from the hardware if the job has partially -+ * completed -+ * @end_timestamp: Time of completion -+ */ -+void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, -+ u32 completion_code, -+ u64 job_tail, -+ ktime_t *end_timestamp); -+ -+/** -+ * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer -+ * -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * @idx: Index into ringbuffer. 0 is the job currently running on -+ * the slot, 1 is the job waiting, all other values are invalid. -+ * Return: The atom at that position in the ringbuffer -+ * or NULL if no atom present -+ */ -+struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, -+ int idx); -+ -+/** -+ * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers -+ * -+ * @kbdev: Device pointer -+ */ -+void kbase_gpu_dump_slots(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_HWACCESS_GPU_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c -new file mode 100755 -index 000000000..54d8ddd80 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.c -@@ -0,0 +1,303 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Base kernel affinity manager APIs -+ */ -+ -+#include -+#include "mali_kbase_js_affinity.h" -+#include "mali_kbase_hw.h" -+ -+#include -+ -+ -+bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, -+ int js) -+{ -+ /* -+ * Here are the reasons for using job slot 2: -+ * - BASE_HW_ISSUE_8987 (which is entirely used for that purpose) -+ * - In absence of the above, then: -+ * - Atoms with BASE_JD_REQ_COHERENT_GROUP -+ * - But, only when there aren't contexts with -+ * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on -+ * all cores on slot 1 could be blocked by those using a coherent group -+ * on slot 2 -+ * - And, only when you actually have 2 or more coregroups - if you -+ * only have 1 coregroup, then having jobs for slot 2 implies they'd -+ * also be for slot 1, meaning you'll get interference from them. Jobs -+ * able to run on slot 2 could also block jobs that can only run on -+ * slot 1 (tiler jobs) -+ */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) -+ return true; -+ -+ if (js != 2) -+ return true; -+ -+ /* Only deal with js==2 now: */ -+ if (kbdev->gpu_props.num_core_groups > 1) { -+ /* Only use slot 2 in the 2+ coregroup case */ -+ if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, -+ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) == -+ false) { -+ /* ...But only when we *don't* have atoms that run on -+ * all cores */ -+ -+ /* No specific check for BASE_JD_REQ_COHERENT_GROUP -+ * atoms - the policy will sort that out */ -+ return true; -+ } -+ } -+ -+ /* Above checks failed mean we shouldn't use slot 2 */ -+ return false; -+} -+ -+/* -+ * As long as it has been decided to have a deeper modification of -+ * what job scheduler, power manager and affinity manager will -+ * implement, this function is just an intermediate step that -+ * assumes: -+ * - all working cores will be powered on when this is called. -+ * - largest current configuration is 2 core groups. -+ * - It has been decided not to have hardcoded values so the low -+ * and high cores in a core split will be evently distributed. -+ * - Odd combinations of core requirements have been filtered out -+ * and do not get to this function (e.g. CS+T+NSS is not -+ * supported here). -+ * - This function is frequently called and can be optimized, -+ * (see notes in loops), but as the functionallity will likely -+ * be modified, optimization has not been addressed. -+*/ -+bool kbase_js_choose_affinity(u64 * const affinity, -+ struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, int js) -+{ -+ base_jd_core_req core_req = katom->core_req; -+ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; -+ u64 core_availability_mask; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ core_availability_mask = kbase_pm_ca_get_core_mask(kbdev); -+ -+ /* -+ * If no cores are currently available (core availability policy is -+ * transitioning) then fail. -+ */ -+ if (0 == core_availability_mask) { -+ *affinity = 0; -+ return false; -+ } -+ -+ KBASE_DEBUG_ASSERT(js >= 0); -+ -+ if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == -+ BASE_JD_REQ_T) { -+ /* If the hardware supports XAFFINITY then we'll only enable -+ * the tiler (which is the default so this is a no-op), -+ * otherwise enable shader core 0. */ -+ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) -+ *affinity = 1; -+ else -+ *affinity = 0; -+ -+ return true; -+ } -+ -+ if (1 == kbdev->gpu_props.num_cores) { -+ /* trivial case only one core, nothing to do */ -+ *affinity = core_availability_mask & -+ kbdev->pm.debug_core_mask[js]; -+ } else { -+ if ((core_req & (BASE_JD_REQ_COHERENT_GROUP | -+ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { -+ if (js == 0 || num_core_groups == 1) { -+ /* js[0] and single-core-group systems just get -+ * the first core group */ -+ *affinity = -+ kbdev->gpu_props.props.coherency_info.group[0].core_mask -+ & core_availability_mask & -+ kbdev->pm.debug_core_mask[js]; -+ } else { -+ /* js[1], js[2] use core groups 0, 1 for -+ * dual-core-group systems */ -+ u32 core_group_idx = ((u32) js) - 1; -+ -+ KBASE_DEBUG_ASSERT(core_group_idx < -+ num_core_groups); -+ *affinity = -+ kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask -+ & core_availability_mask & -+ kbdev->pm.debug_core_mask[js]; -+ -+ /* If the job is specifically targeting core -+ * group 1 and the core availability policy is -+ * keeping that core group off, then fail */ -+ if (*affinity == 0 && core_group_idx == 1 && -+ kbdev->pm.backend.cg1_disabled -+ == true) -+ katom->event_code = -+ BASE_JD_EVENT_PM_EVENT; -+ } -+ } else { -+ /* All cores are available when no core split is -+ * required */ -+ *affinity = core_availability_mask & -+ kbdev->pm.debug_core_mask[js]; -+ } -+ } -+ -+ /* -+ * If no cores are currently available in the desired core group(s) -+ * (core availability policy is transitioning) then fail. -+ */ -+ if (*affinity == 0) -+ return false; -+ -+ /* Enable core 0 if tiler required for hardware without XAFFINITY -+ * support (notes above) */ -+ if (core_req & BASE_JD_REQ_T) { -+ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) -+ *affinity = *affinity | 1; -+ } -+ -+ return true; -+} -+ -+static inline bool kbase_js_affinity_is_violating( -+ struct kbase_device *kbdev, -+ u64 *affinities) -+{ -+ /* This implementation checks whether the two slots involved in Generic -+ * thread creation have intersecting affinity. This is due to micro- -+ * architectural issues where a job in slot A targetting cores used by -+ * slot B could prevent the job in slot B from making progress until the -+ * job in slot A has completed. -+ */ -+ u64 affinity_set_left; -+ u64 affinity_set_right; -+ u64 intersection; -+ -+ KBASE_DEBUG_ASSERT(affinities != NULL); -+ -+ affinity_set_left = affinities[1]; -+ -+ affinity_set_right = affinities[2]; -+ -+ /* A violation occurs when any bit in the left_set is also in the -+ * right_set */ -+ intersection = affinity_set_left & affinity_set_right; -+ -+ return (bool) (intersection != (u64) 0u); -+} -+ -+bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, -+ u64 affinity) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ u64 new_affinities[BASE_JM_MAX_NR_SLOTS]; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); -+ js_devdata = &kbdev->js_data; -+ -+ memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities, -+ sizeof(js_devdata->runpool_irq.slot_affinities)); -+ -+ new_affinities[js] |= affinity; -+ -+ return kbase_js_affinity_is_violating(kbdev, new_affinities); -+} -+ -+void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, -+ u64 affinity) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ u64 cores; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); -+ js_devdata = &kbdev->js_data; -+ -+ KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity) -+ == false); -+ -+ cores = affinity; -+ while (cores) { -+ int bitnum = fls64(cores) - 1; -+ u64 bit = 1ULL << bitnum; -+ s8 cnt; -+ -+ cnt = -+ ++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); -+ -+ if (cnt == 1) -+ js_devdata->runpool_irq.slot_affinities[js] |= bit; -+ -+ cores &= ~bit; -+ } -+} -+ -+void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, -+ u64 affinity) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ u64 cores; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS); -+ js_devdata = &kbdev->js_data; -+ -+ cores = affinity; -+ while (cores) { -+ int bitnum = fls64(cores) - 1; -+ u64 bit = 1ULL << bitnum; -+ s8 cnt; -+ -+ KBASE_DEBUG_ASSERT( -+ js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0); -+ -+ cnt = -+ --(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]); -+ -+ if (0 == cnt) -+ js_devdata->runpool_irq.slot_affinities[js] &= ~bit; -+ -+ cores &= ~bit; -+ } -+} -+ -+#if KBASE_TRACE_ENABLE -+void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ int slot_nr; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ js_devdata = &kbdev->js_data; -+ -+ for (slot_nr = 0; slot_nr < 3; ++slot_nr) -+ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL, -+ NULL, 0u, slot_nr, -+ (u32) js_devdata->runpool_irq.slot_affinities[slot_nr]); -+} -+#endif /* KBASE_TRACE_ENABLE */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h -new file mode 100755 -index 000000000..35d9781ae ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_affinity.h -@@ -0,0 +1,129 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Affinity Manager internal APIs. -+ */ -+ -+#ifndef _KBASE_JS_AFFINITY_H_ -+#define _KBASE_JS_AFFINITY_H_ -+ -+/** -+ * kbase_js_can_run_job_on_slot_no_lock - Decide whether it is possible to -+ * submit a job to a particular job slot in the current status -+ * -+ * @kbdev: The kbase device structure of the device -+ * @js: Job slot number to check for allowance -+ * -+ * Will check if submitting to the given job slot is allowed in the current -+ * status. For example using job slot 2 while in soft-stoppable state and only -+ * having 1 coregroup is not allowed by the policy. This function should be -+ * called prior to submitting a job to a slot to make sure policy rules are not -+ * violated. -+ * -+ * The following locking conditions are made on the caller -+ * - it must hold hwaccess_lock -+ */ -+bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev, int js); -+ -+/** -+ * kbase_js_choose_affinity - Compute affinity for a given job. -+ * -+ * @affinity: Affinity bitmap computed -+ * @kbdev: The kbase device structure of the device -+ * @katom: Job chain of which affinity is going to be found -+ * @js: Slot the job chain is being submitted -+ * -+ * Currently assumes an all-on/all-off power management policy. -+ * Also assumes there is at least one core with tiler available. -+ * -+ * Returns true if a valid affinity was chosen, false if -+ * no cores were available. -+ */ -+bool kbase_js_choose_affinity(u64 * const affinity, -+ struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, -+ int js); -+ -+/** -+ * kbase_js_affinity_would_violate - Determine whether a proposed affinity on -+ * job slot @js would cause a violation of affinity restrictions. -+ * -+ * @kbdev: Kbase device structure -+ * @js: The job slot to test -+ * @affinity: The affinity mask to test -+ * -+ * The following locks must be held by the caller -+ * - hwaccess_lock -+ * -+ * Return: true if the affinity would violate the restrictions -+ */ -+bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js, -+ u64 affinity); -+ -+/** -+ * kbase_js_affinity_retain_slot_cores - Affinity tracking: retain cores used by -+ * a slot -+ * -+ * @kbdev: Kbase device structure -+ * @js: The job slot retaining the cores -+ * @affinity: The cores to retain -+ * -+ * The following locks must be held by the caller -+ * - hwaccess_lock -+ */ -+void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js, -+ u64 affinity); -+ -+/** -+ * kbase_js_affinity_release_slot_cores - Affinity tracking: release cores used -+ * by a slot -+ * -+ * @kbdev: Kbase device structure -+ * @js: Job slot -+ * @affinity: Bit mask of core to be released -+ * -+ * Cores must be released as soon as a job is dequeued from a slot's 'submit -+ * slots', and before another job is submitted to those slots. Otherwise, the -+ * refcount could exceed the maximum number submittable to a slot, -+ * %BASE_JM_SUBMIT_SLOTS. -+ * -+ * The following locks must be held by the caller -+ * - hwaccess_lock -+ */ -+void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js, -+ u64 affinity); -+ -+/** -+ * kbase_js_debug_log_current_affinities - log the current affinities -+ * -+ * @kbdev: Kbase device structure -+ * -+ * Output to the Trace log the current tracked affinities on all slots -+ */ -+#if KBASE_TRACE_ENABLE -+void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev); -+#else /* KBASE_TRACE_ENABLE */ -+static inline void -+kbase_js_debug_log_current_affinities(struct kbase_device *kbdev) -+{ -+} -+#endif /* KBASE_TRACE_ENABLE */ -+ -+#endif /* _KBASE_JS_AFFINITY_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c -new file mode 100755 -index 000000000..a8c1af23a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c -@@ -0,0 +1,356 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register-based HW access backend specific job scheduler APIs -+ */ -+ -+#include -+#include -+#include -+#include -+ -+/* -+ * Define for when dumping is enabled. -+ * This should not be based on the instrumentation level as whether dumping is -+ * enabled for a particular level is down to the integrator. However this is -+ * being used for now as otherwise the cinstr headers would be needed. -+ */ -+#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) -+ -+/* -+ * Hold the runpool_mutex for this -+ */ -+static inline bool timer_callback_should_run(struct kbase_device *kbdev) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ s8 nr_running_ctxs; -+ -+ lockdep_assert_held(&kbdev->js_data.runpool_mutex); -+ -+ /* Timer must stop if we are suspending */ -+ if (backend->suspend_timer) -+ return false; -+ -+ /* nr_contexts_pullable is updated with the runpool_mutex. However, the -+ * locking in the caller gives us a barrier that ensures -+ * nr_contexts_pullable is up-to-date for reading */ -+ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); -+ -+#ifdef CONFIG_MALI_DEBUG -+ if (kbdev->js_data.softstop_always) { -+ /* Debug support for allowing soft-stop on a single context */ -+ return true; -+ } -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { -+ /* Timeouts would have to be 4x longer (due to micro- -+ * architectural design) to support OpenCL conformance tests, so -+ * only run the timer when there's: -+ * - 2 or more CL contexts -+ * - 1 or more GLES contexts -+ * -+ * NOTE: We will treat a context that has both Compute and Non- -+ * Compute jobs will be treated as an OpenCL context (hence, we -+ * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). -+ */ -+ { -+ s8 nr_compute_ctxs = -+ kbasep_js_ctx_attr_count_on_runpool(kbdev, -+ KBASEP_JS_CTX_ATTR_COMPUTE); -+ s8 nr_noncompute_ctxs = nr_running_ctxs - -+ nr_compute_ctxs; -+ -+ return (bool) (nr_compute_ctxs >= 2 || -+ nr_noncompute_ctxs > 0); -+ } -+ } else { -+ /* Run the timer callback whenever you have at least 1 context -+ */ -+ return (bool) (nr_running_ctxs > 0); -+ } -+} -+ -+static enum hrtimer_restart timer_callback(struct hrtimer *timer) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev; -+ struct kbasep_js_device_data *js_devdata; -+ struct kbase_backend_data *backend; -+ int s; -+ bool reset_needed = false; -+ -+ KBASE_DEBUG_ASSERT(timer != NULL); -+ -+ backend = container_of(timer, struct kbase_backend_data, -+ scheduling_timer); -+ kbdev = container_of(backend, struct kbase_device, hwaccess.backend); -+ js_devdata = &kbdev->js_data; -+ -+ /* Loop through the slots */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { -+ struct kbase_jd_atom *atom = NULL; -+ -+ if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { -+ atom = kbase_gpu_inspect(kbdev, s, 0); -+ KBASE_DEBUG_ASSERT(atom != NULL); -+ } -+ -+ if (atom != NULL) { -+ /* The current version of the model doesn't support -+ * Soft-Stop */ -+ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { -+ u32 ticks = atom->ticks++; -+ -+#if !CINSTR_DUMPING_ENABLED -+ u32 soft_stop_ticks, hard_stop_ticks, -+ gpu_reset_ticks; -+ if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { -+ soft_stop_ticks = -+ js_devdata->soft_stop_ticks_cl; -+ hard_stop_ticks = -+ js_devdata->hard_stop_ticks_cl; -+ gpu_reset_ticks = -+ js_devdata->gpu_reset_ticks_cl; -+ } else { -+ soft_stop_ticks = -+ js_devdata->soft_stop_ticks; -+ hard_stop_ticks = -+ js_devdata->hard_stop_ticks_ss; -+ gpu_reset_ticks = -+ js_devdata->gpu_reset_ticks_ss; -+ } -+ -+ /* If timeouts have been changed then ensure -+ * that atom tick count is not greater than the -+ * new soft_stop timeout. This ensures that -+ * atoms do not miss any of the timeouts due to -+ * races between this worker and the thread -+ * changing the timeouts. */ -+ if (backend->timeouts_updated && -+ ticks > soft_stop_ticks) -+ ticks = atom->ticks = soft_stop_ticks; -+ -+ /* Job is Soft-Stoppable */ -+ if (ticks == soft_stop_ticks) { -+ int disjoint_threshold = -+ KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; -+ u32 softstop_flags = 0u; -+ /* Job has been scheduled for at least -+ * js_devdata->soft_stop_ticks ticks. -+ * Soft stop the slot so we can run -+ * other jobs. -+ */ -+ dev_dbg(kbdev->dev, "Soft-stop"); -+#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS -+ /* nr_user_contexts_running is updated -+ * with the runpool_mutex, but we can't -+ * take that here. -+ * -+ * However, if it's about to be -+ * increased then the new context can't -+ * run any jobs until they take the -+ * hwaccess_lock, so it's OK to observe -+ * the older value. -+ * -+ * Similarly, if it's about to be -+ * decreased, the last job from another -+ * context has already finished, so it's -+ * not too bad that we observe the older -+ * value and register a disjoint event -+ * when we try soft-stopping */ -+ if (js_devdata->nr_user_contexts_running -+ >= disjoint_threshold) -+ softstop_flags |= -+ JS_COMMAND_SW_CAUSES_DISJOINT; -+ -+ kbase_job_slot_softstop_swflags(kbdev, -+ s, atom, softstop_flags); -+#endif -+ } else if (ticks == hard_stop_ticks) { -+ /* Job has been scheduled for at least -+ * js_devdata->hard_stop_ticks_ss ticks. -+ * It should have been soft-stopped by -+ * now. Hard stop the slot. -+ */ -+#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS -+ int ms = -+ js_devdata->scheduling_period_ns -+ / 1000000u; -+ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", -+ (unsigned long)ticks, -+ (unsigned long)ms); -+ kbase_job_slot_hardstop(atom->kctx, s, -+ atom); -+#endif -+ } else if (ticks == gpu_reset_ticks) { -+ /* Job has been scheduled for at least -+ * js_devdata->gpu_reset_ticks_ss ticks. -+ * It should have left the GPU by now. -+ * Signal that the GPU needs to be -+ * reset. -+ */ -+ reset_needed = true; -+ } -+#else /* !CINSTR_DUMPING_ENABLED */ -+ /* NOTE: During CINSTR_DUMPING_ENABLED, we use -+ * the alternate timeouts, which makes the hard- -+ * stop and GPU reset timeout much longer. We -+ * also ensure that we don't soft-stop at all. -+ */ -+ if (ticks == js_devdata->soft_stop_ticks) { -+ /* Job has been scheduled for at least -+ * js_devdata->soft_stop_ticks. We do -+ * not soft-stop during -+ * CINSTR_DUMPING_ENABLED, however. -+ */ -+ dev_dbg(kbdev->dev, "Soft-stop"); -+ } else if (ticks == -+ js_devdata->hard_stop_ticks_dumping) { -+ /* Job has been scheduled for at least -+ * js_devdata->hard_stop_ticks_dumping -+ * ticks. Hard stop the slot. -+ */ -+#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS -+ int ms = -+ js_devdata->scheduling_period_ns -+ / 1000000u; -+ dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", -+ (unsigned long)ticks, -+ (unsigned long)ms); -+ kbase_job_slot_hardstop(atom->kctx, s, -+ atom); -+#endif -+ } else if (ticks == -+ js_devdata->gpu_reset_ticks_dumping) { -+ /* Job has been scheduled for at least -+ * js_devdata->gpu_reset_ticks_dumping -+ * ticks. It should have left the GPU by -+ * now. Signal that the GPU needs to be -+ * reset. -+ */ -+ reset_needed = true; -+ } -+#endif /* !CINSTR_DUMPING_ENABLED */ -+ } -+ } -+ } -+#if KBASE_GPU_RESET_EN -+ if (reset_needed) { -+ dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); -+ -+ if (kbase_prepare_to_reset_gpu_locked(kbdev)) -+ kbase_reset_gpu_locked(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ /* the timer is re-issued if there is contexts in the run-pool */ -+ -+ if (backend->timer_running) -+ hrtimer_start(&backend->scheduling_timer, -+ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), -+ HRTIMER_MODE_REL); -+ -+ backend->timeouts_updated = false; -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return HRTIMER_NORESTART; -+} -+ -+void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ unsigned long flags; -+ -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ -+ if (!timer_callback_should_run(kbdev)) { -+ /* Take spinlock to force synchronisation with timer */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ backend->timer_running = false; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ /* From now on, return value of timer_callback_should_run() will -+ * also cause the timer to not requeue itself. Its return value -+ * cannot change, because it depends on variables updated with -+ * the runpool_mutex held, which the caller of this must also -+ * hold */ -+ hrtimer_cancel(&backend->scheduling_timer); -+ } -+ -+ if (timer_callback_should_run(kbdev) && !backend->timer_running) { -+ /* Take spinlock to force synchronisation with timer */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ backend->timer_running = true; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ hrtimer_start(&backend->scheduling_timer, -+ HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), -+ HRTIMER_MODE_REL); -+ -+ KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, -+ 0u); -+ } -+} -+ -+int kbase_backend_timer_init(struct kbase_device *kbdev) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ -+ hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, -+ HRTIMER_MODE_REL); -+ backend->scheduling_timer.function = timer_callback; -+ -+ backend->timer_running = false; -+ -+ return 0; -+} -+ -+void kbase_backend_timer_term(struct kbase_device *kbdev) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ -+ hrtimer_cancel(&backend->scheduling_timer); -+} -+ -+void kbase_backend_timer_suspend(struct kbase_device *kbdev) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ -+ backend->suspend_timer = true; -+ -+ kbase_backend_ctx_count_changed(kbdev); -+} -+ -+void kbase_backend_timer_resume(struct kbase_device *kbdev) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ -+ backend->suspend_timer = false; -+ -+ kbase_backend_ctx_count_changed(kbdev); -+} -+ -+void kbase_backend_timeouts_changed(struct kbase_device *kbdev) -+{ -+ struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -+ -+ backend->timeouts_updated = true; -+} -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h -new file mode 100755 -index 000000000..3f53779c6 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h -@@ -0,0 +1,69 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Register-based HW access backend specific job scheduler APIs -+ */ -+ -+#ifndef _KBASE_JS_BACKEND_H_ -+#define _KBASE_JS_BACKEND_H_ -+ -+/** -+ * kbase_backend_timer_init() - Initialise the JS scheduling timer -+ * @kbdev: Device pointer -+ * -+ * This function should be called at driver initialisation -+ * -+ * Return: 0 on success -+ */ -+int kbase_backend_timer_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_timer_term() - Terminate the JS scheduling timer -+ * @kbdev: Device pointer -+ * -+ * This function should be called at driver termination -+ */ -+void kbase_backend_timer_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling -+ * timer -+ * @kbdev: Device pointer -+ * -+ * This function should be called on suspend, after the active count has reached -+ * zero. This is required as the timer may have been started on job submission -+ * to the job scheduler, but before jobs are submitted to the GPU. -+ * -+ * Caller must hold runpool_mutex. -+ */ -+void kbase_backend_timer_suspend(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS -+ * scheduling timer -+ * @kbdev: Device pointer -+ * -+ * This function should be called on resume. Note that is is not guaranteed to -+ * re-start the timer, only evalute whether it should be re-started. -+ * -+ * Caller must hold runpool_mutex. -+ */ -+void kbase_backend_timer_resume(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_JS_BACKEND_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c -new file mode 100755 -index 000000000..ba826184d ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.c -@@ -0,0 +1,407 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+/* #define ENABLE_DEBUG_LOG */ -+#include "../../platform/rk/custom_log.h" -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, -+ u32 num_pages) -+{ -+ u64 region; -+ -+ /* can't lock a zero sized range */ -+ KBASE_DEBUG_ASSERT(num_pages); -+ -+ region = pfn << PAGE_SHIFT; -+ /* -+ * fls returns (given the ASSERT above): -+ * 1 .. 32 -+ * -+ * 10 + fls(num_pages) -+ * results in the range (11 .. 42) -+ */ -+ -+ /* gracefully handle num_pages being zero */ -+ if (0 == num_pages) { -+ region |= 11; -+ } else { -+ u8 region_width; -+ -+ region_width = 10 + fls(num_pages); -+ if (num_pages != (1ul << (region_width - 11))) { -+ /* not pow2, so must go up to the next pow2 */ -+ region_width += 1; -+ } -+ KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE); -+ KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE); -+ region |= region_width; -+ } -+ -+ return region; -+} -+ -+static int wait_ready(struct kbase_device *kbdev, -+ unsigned int as_nr, struct kbase_context *kctx) -+{ -+ unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; -+ u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); -+ -+ /* Wait for the MMU status to indicate there is no active command, in -+ * case one is pending. Do not log remaining register accesses. */ -+ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) -+ val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), NULL); -+ -+ if (max_loops == 0) { -+ dev_err(kbdev->dev, "AS_ACTIVE bit stuck\n"); -+ return -1; -+ } -+ -+ /* If waiting in loop was performed, log last read value. */ -+ if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) -+ kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS), kctx); -+ -+ return 0; -+} -+ -+static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd, -+ struct kbase_context *kctx) -+{ -+ int status; -+ -+ /* write AS_COMMAND when MMU is ready to accept another command */ -+ status = wait_ready(kbdev, as_nr, kctx); -+ if (status == 0) -+ kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd, -+ kctx); -+ -+ return status; -+} -+ -+static void validate_protected_page_fault(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ /* GPUs which support (native) protected mode shall not report page -+ * fault addresses unless it has protected debug mode and protected -+ * debug mode is turned on */ -+ u32 protected_debug_mode = 0; -+ -+ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) -+ return; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { -+ protected_debug_mode = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_STATUS), -+ kctx) & GPU_DBGEN; -+ } -+ -+ if (!protected_debug_mode) { -+ /* fault_addr should never be reported in protected mode. -+ * However, we just continue by printing an error message */ -+ dev_err(kbdev->dev, "Fault address reported in protected mode\n"); -+ } -+} -+ -+void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) -+{ -+ const int num_as = 16; -+ const int busfault_shift = MMU_PAGE_FAULT_FLAGS; -+ const int pf_shift = 0; -+ const unsigned long as_bit_mask = (1UL << num_as) - 1; -+ unsigned long flags; -+ u32 new_mask; -+ u32 tmp; -+ -+ /* bus faults */ -+ u32 bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; -+ /* page faults (note: Ignore ASes with both pf and bf) */ -+ u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ -+ /* remember current mask */ -+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); -+ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); -+ /* mask interrupts for now */ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); -+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); -+ -+ while (bf_bits | pf_bits) { -+ struct kbase_as *as; -+ int as_no; -+ struct kbase_context *kctx; -+ -+ /* -+ * the while logic ensures we have a bit set, no need to check -+ * for not-found here -+ */ -+ as_no = ffs(bf_bits | pf_bits) - 1; -+ as = &kbdev->as[as_no]; -+ -+ /* -+ * Refcount the kctx ASAP - it shouldn't disappear anyway, since -+ * Bus/Page faults _should_ only occur whilst jobs are running, -+ * and a job causing the Bus/Page fault shouldn't complete until -+ * the MMU is updated -+ */ -+ kctx = kbasep_js_runpool_lookup_ctx(kbdev, as_no); -+ if (!kctx) { -+ E("fail to lookup ctx, to break out."); -+ break; -+ } -+ -+ -+ /* find faulting address */ -+ as->fault_addr = kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, -+ AS_FAULTADDRESS_HI), -+ kctx); -+ as->fault_addr <<= 32; -+ as->fault_addr |= kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, -+ AS_FAULTADDRESS_LO), -+ kctx); -+ -+ /* Mark the fault protected or not */ -+ as->protected_mode = kbdev->protected_mode; -+ -+ if (kbdev->protected_mode && as->fault_addr) -+ { -+ /* check if address reporting is allowed */ -+ validate_protected_page_fault(kbdev, kctx); -+ } -+ -+ /* report the fault to debugfs */ -+ kbase_as_fault_debugfs_new(kbdev, as_no); -+ -+ /* record the fault status */ -+ as->fault_status = kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, -+ AS_FAULTSTATUS), -+ kctx); -+ -+ /* find the fault type */ -+ as->fault_type = (bf_bits & (1 << as_no)) ? -+ KBASE_MMU_FAULT_TYPE_BUS : -+ KBASE_MMU_FAULT_TYPE_PAGE; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { -+ as->fault_extra_addr = kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI), -+ kctx); -+ as->fault_extra_addr <<= 32; -+ as->fault_extra_addr |= kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO), -+ kctx); -+ } -+ -+ if (kbase_as_has_bus_fault(as)) { -+ /* Mark bus fault as handled. -+ * Note that a bus fault is processed first in case -+ * where both a bus fault and page fault occur. -+ */ -+ bf_bits &= ~(1UL << as_no); -+ -+ /* remove the queued BF (and PF) from the mask */ -+ new_mask &= ~(MMU_BUS_ERROR(as_no) | -+ MMU_PAGE_FAULT(as_no)); -+ } else { -+ /* Mark page fault as handled */ -+ pf_bits &= ~(1UL << as_no); -+ -+ /* remove the queued PF from the mask */ -+ new_mask &= ~MMU_PAGE_FAULT(as_no); -+ } -+ -+ /* Process the interrupt for this address space */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_interrupt_process(kbdev, kctx, as); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+ -+ /* reenable interrupts */ -+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); -+ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL); -+ new_mask |= tmp; -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask, NULL); -+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); -+} -+ -+void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx) -+{ -+ struct kbase_mmu_setup *current_setup = &as->current_setup; -+ u32 transcfg = 0; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { -+ transcfg = current_setup->transcfg & 0xFFFFFFFFUL; -+ -+ /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK */ -+ /* Clear PTW_MEMATTR bits */ -+ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; -+ /* Enable correct PTW_MEMATTR bits */ -+ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; -+ -+ if (kbdev->system_coherency == COHERENCY_ACE) { -+ /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) */ -+ /* Clear PTW_SH bits */ -+ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); -+ /* Enable correct PTW_SH bits */ -+ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); -+ } -+ -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), -+ transcfg, kctx); -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), -+ (current_setup->transcfg >> 32) & 0xFFFFFFFFUL, -+ kctx); -+ } else { -+ if (kbdev->system_coherency == COHERENCY_ACE) -+ current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; -+ } -+ -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), -+ current_setup->transtab & 0xFFFFFFFFUL, kctx); -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), -+ (current_setup->transtab >> 32) & 0xFFFFFFFFUL, kctx); -+ -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), -+ current_setup->memattr & 0xFFFFFFFFUL, kctx); -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), -+ (current_setup->memattr >> 32) & 0xFFFFFFFFUL, kctx); -+ -+ KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, -+ current_setup->transtab, -+ current_setup->memattr, -+ transcfg); -+ -+ write_cmd(kbdev, as->number, AS_COMMAND_UPDATE, kctx); -+} -+ -+int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 op, -+ unsigned int handling_irq) -+{ -+ int ret; -+ -+ lockdep_assert_held(&kbdev->mmu_hw_mutex); -+ -+ if (op == AS_COMMAND_UNLOCK) { -+ /* Unlock doesn't require a lock first */ -+ ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); -+ } else { -+ u64 lock_addr = lock_region(kbdev, vpfn, nr); -+ -+ /* Lock the region that needs to be updated */ -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO), -+ lock_addr & 0xFFFFFFFFUL, kctx); -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI), -+ (lock_addr >> 32) & 0xFFFFFFFFUL, kctx); -+ write_cmd(kbdev, as->number, AS_COMMAND_LOCK, kctx); -+ -+ /* Run the MMU operation */ -+ write_cmd(kbdev, as->number, op, kctx); -+ -+ /* Wait for the flush to complete */ -+ ret = wait_ready(kbdev, as->number, kctx); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) { -+ /* Issue an UNLOCK command to ensure that valid page -+ tables are re-read by the GPU after an update. -+ Note that, the FLUSH command should perform all the -+ actions necessary, however the bus logs show that if -+ multiple page faults occur within an 8 page region -+ the MMU does not always re-read the updated page -+ table entries for later faults or is only partially -+ read, it subsequently raises the page fault IRQ for -+ the same addresses, the unlock ensures that the MMU -+ cache is flushed, so updates can be re-read. As the -+ region is now unlocked we need to issue 2 UNLOCK -+ commands in order to flush the MMU/uTLB, -+ see PRLAM-8812. -+ */ -+ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); -+ write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK, kctx); -+ } -+ } -+ -+ return ret; -+} -+ -+void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx, enum kbase_mmu_fault_type type) -+{ -+ unsigned long flags; -+ u32 pf_bf_mask; -+ -+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); -+ -+ /* -+ * A reset is in-flight and we're flushing the IRQ + bottom half -+ * so don't update anything as it could race with the reset code. -+ */ -+ if (kbdev->irq_reset_flush) -+ goto unlock; -+ -+ /* Clear the page (and bus fault IRQ as well in case one occurred) */ -+ pf_bf_mask = MMU_PAGE_FAULT(as->number); -+ if (type == KBASE_MMU_FAULT_TYPE_BUS || -+ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) -+ pf_bf_mask |= MMU_BUS_ERROR(as->number); -+ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask, kctx); -+ -+unlock: -+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); -+} -+ -+void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx, enum kbase_mmu_fault_type type) -+{ -+ unsigned long flags; -+ u32 irq_mask; -+ -+ /* Enable the page fault IRQ (and bus fault IRQ as well in case one -+ * occurred) */ -+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); -+ -+ /* -+ * A reset is in-flight and we're flushing the IRQ + bottom half -+ * so don't update anything as it could race with the reset code. -+ */ -+ if (kbdev->irq_reset_flush) -+ goto unlock; -+ -+ irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), kctx) | -+ MMU_PAGE_FAULT(as->number); -+ -+ if (type == KBASE_MMU_FAULT_TYPE_BUS || -+ type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) -+ irq_mask |= MMU_BUS_ERROR(as->number); -+ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask, kctx); -+ -+unlock: -+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); -+} -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h -new file mode 100755 -index 000000000..c02253c6a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_mmu_hw_direct.h -@@ -0,0 +1,42 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Interface file for the direct implementation for MMU hardware access -+ * -+ * Direct MMU hardware interface -+ * -+ * This module provides the interface(s) that are required by the direct -+ * register access implementation of the MMU hardware interface -+ */ -+ -+#ifndef _MALI_KBASE_MMU_HW_DIRECT_H_ -+#define _MALI_KBASE_MMU_HW_DIRECT_H_ -+ -+#include -+ -+/** -+ * kbase_mmu_interrupt - Process an MMU interrupt. -+ * -+ * Process the MMU interrupt that was reported by the &kbase_device. -+ * -+ * @kbdev: kbase context to clear the fault from. -+ * @irq_stat: Value of the MMU_IRQ_STATUS register -+ */ -+void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); -+ -+#endif /* _MALI_KBASE_MMU_HW_DIRECT_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c -new file mode 100755 -index 000000000..0614348e9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c -@@ -0,0 +1,63 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * "Always on" power management policy -+ */ -+ -+#include -+#include -+ -+static u64 always_on_get_core_mask(struct kbase_device *kbdev) -+{ -+ return kbdev->gpu_props.props.raw_props.shader_present; -+} -+ -+static bool always_on_get_core_active(struct kbase_device *kbdev) -+{ -+ return true; -+} -+ -+static void always_on_init(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+static void always_on_term(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+/* -+ * The struct kbase_pm_policy structure for the demand power policy. -+ * -+ * This is the static structure that defines the demand power policy's callback -+ * and name. -+ */ -+const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { -+ "always_on", /* name */ -+ always_on_init, /* init */ -+ always_on_term, /* term */ -+ always_on_get_core_mask, /* get_core_mask */ -+ always_on_get_core_active, /* get_core_active */ -+ 0u, /* flags */ -+ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ -+}; -+ -+KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h -new file mode 100755 -index 000000000..f9d244b01 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h -@@ -0,0 +1,77 @@ -+ -+/* -+ * -+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * "Always on" power management policy -+ */ -+ -+#ifndef MALI_KBASE_PM_ALWAYS_ON_H -+#define MALI_KBASE_PM_ALWAYS_ON_H -+ -+/** -+ * DOC: -+ * The "Always on" power management policy has the following -+ * characteristics: -+ * -+ * - When KBase indicates that the GPU will be powered up, but we don't yet -+ * know which Job Chains are to be run: -+ * All Shader Cores are powered up, regardless of whether or not they will -+ * be needed later. -+ * -+ * - When KBase indicates that a set of Shader Cores are needed to submit the -+ * currently queued Job Chains: -+ * All Shader Cores are kept powered, regardless of whether or not they will -+ * be needed -+ * -+ * - When KBase indicates that the GPU need not be powered: -+ * The Shader Cores are kept powered, regardless of whether or not they will -+ * be needed. The GPU itself is also kept powered, even though it is not -+ * needed. -+ * -+ * This policy is automatically overridden during system suspend: the desired -+ * core state is ignored, and the cores are forced off regardless of what the -+ * policy requests. After resuming from suspend, new changes to the desired -+ * core state made by the policy are honored. -+ * -+ * Note: -+ * -+ * - KBase indicates the GPU will be powered up when it has a User Process that -+ * has just started to submit Job Chains. -+ * -+ * - KBase indicates the GPU need not be powered when all the Job Chains from -+ * User Processes have finished, and it is waiting for a User Process to -+ * submit some more Job Chains. -+ */ -+ -+/** -+ * struct kbasep_pm_policy_always_on - Private struct for policy instance data -+ * @dummy: unused dummy variable -+ * -+ * This contains data that is private to the particular power policy that is -+ * active. -+ */ -+struct kbasep_pm_policy_always_on { -+ int dummy; -+}; -+ -+extern const struct kbase_pm_policy kbase_pm_always_on_policy_ops; -+ -+#endif /* MALI_KBASE_PM_ALWAYS_ON_H */ -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c -new file mode 100755 -index 000000000..146fd48ba ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c -@@ -0,0 +1,482 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * GPU backend implementation of base kernel power management APIs -+ */ -+ -+#include -+#include -+#include -+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE -+#include -+#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); -+ -+void kbase_pm_register_access_enable(struct kbase_device *kbdev) -+{ -+ struct kbase_pm_callback_conf *callbacks; -+ -+ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; -+ -+ if (callbacks) -+ callbacks->power_on_callback(kbdev); -+ -+ kbdev->pm.backend.gpu_powered = true; -+} -+ -+void kbase_pm_register_access_disable(struct kbase_device *kbdev) -+{ -+ struct kbase_pm_callback_conf *callbacks; -+ -+ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; -+ -+ if (callbacks) -+ callbacks->power_off_callback(kbdev); -+ -+ kbdev->pm.backend.gpu_powered = false; -+} -+ -+int kbase_hwaccess_pm_init(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ struct kbase_pm_callback_conf *callbacks; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ mutex_init(&kbdev->pm.lock); -+ -+ kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", -+ WQ_HIGHPRI | WQ_UNBOUND, 1); -+ if (!kbdev->pm.backend.gpu_poweroff_wait_wq) -+ return -ENOMEM; -+ -+ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, -+ kbase_pm_gpu_poweroff_wait_wq); -+ -+ kbdev->pm.backend.gpu_powered = false; -+ kbdev->pm.suspending = false; -+#ifdef CONFIG_MALI_DEBUG -+ kbdev->pm.backend.driver_ready_for_irqs = false; -+#endif /* CONFIG_MALI_DEBUG */ -+ kbdev->pm.backend.gpu_in_desired_state = true; -+ init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); -+ -+ callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; -+ if (callbacks) { -+ kbdev->pm.backend.callback_power_on = -+ callbacks->power_on_callback; -+ kbdev->pm.backend.callback_power_off = -+ callbacks->power_off_callback; -+ kbdev->pm.backend.callback_power_suspend = -+ callbacks->power_suspend_callback; -+ kbdev->pm.backend.callback_power_resume = -+ callbacks->power_resume_callback; -+ kbdev->pm.callback_power_runtime_init = -+ callbacks->power_runtime_init_callback; -+ kbdev->pm.callback_power_runtime_term = -+ callbacks->power_runtime_term_callback; -+ kbdev->pm.backend.callback_power_runtime_on = -+ callbacks->power_runtime_on_callback; -+ kbdev->pm.backend.callback_power_runtime_off = -+ callbacks->power_runtime_off_callback; -+ kbdev->pm.backend.callback_power_runtime_idle = -+ callbacks->power_runtime_idle_callback; -+ } else { -+ kbdev->pm.backend.callback_power_on = NULL; -+ kbdev->pm.backend.callback_power_off = NULL; -+ kbdev->pm.backend.callback_power_suspend = NULL; -+ kbdev->pm.backend.callback_power_resume = NULL; -+ kbdev->pm.callback_power_runtime_init = NULL; -+ kbdev->pm.callback_power_runtime_term = NULL; -+ kbdev->pm.backend.callback_power_runtime_on = NULL; -+ kbdev->pm.backend.callback_power_runtime_off = NULL; -+ kbdev->pm.backend.callback_power_runtime_idle = NULL; -+ } -+ -+ /* Initialise the metrics subsystem */ -+ ret = kbasep_pm_metrics_init(kbdev); -+ if (ret) -+ return ret; -+ -+ init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait); -+ kbdev->pm.backend.l2_powered = 0; -+ -+ init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); -+ kbdev->pm.backend.reset_done = false; -+ -+ init_waitqueue_head(&kbdev->pm.zero_active_count_wait); -+ kbdev->pm.active_count = 0; -+ -+ spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); -+ spin_lock_init(&kbdev->pm.backend.gpu_powered_lock); -+ -+ init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); -+ -+ if (kbase_pm_ca_init(kbdev) != 0) -+ goto workq_fail; -+ -+ if (kbase_pm_policy_init(kbdev) != 0) -+ goto pm_policy_fail; -+ -+ return 0; -+ -+pm_policy_fail: -+ kbase_pm_ca_term(kbdev); -+workq_fail: -+ kbasep_pm_metrics_term(kbdev); -+ return -EINVAL; -+} -+ -+void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) -+{ -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ /* Turn clocks and interrupts on - no-op if we haven't done a previous -+ * kbase_pm_clock_off() */ -+ kbase_pm_clock_on(kbdev, is_resume); -+ -+ /* Update core status as required by the policy */ -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START); -+ kbase_pm_update_cores_state(kbdev); -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END); -+ -+ /* NOTE: We don't wait to reach the desired state, since running atoms -+ * will wait for that state to be reached anyway */ -+} -+ -+static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) -+{ -+ struct kbase_device *kbdev = container_of(data, struct kbase_device, -+ pm.backend.gpu_poweroff_wait_work); -+ struct kbase_pm_device_data *pm = &kbdev->pm; -+ struct kbase_pm_backend_data *backend = &pm->backend; -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ unsigned long flags; -+ -+#if !PLATFORM_POWER_DOWN_ONLY -+ /* Wait for power transitions to complete. We do this with no locks held -+ * so that we don't deadlock with any pending workqueues */ -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START); -+ kbase_pm_check_transitions_sync(kbdev); -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END); -+#endif /* !PLATFORM_POWER_DOWN_ONLY */ -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+#if PLATFORM_POWER_DOWN_ONLY -+ if (kbdev->pm.backend.gpu_powered) { -+ if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)) { -+ /* If L2 cache is powered then we must flush it before -+ * we power off the GPU. Normally this would have been -+ * handled when the L2 was powered off. */ -+ kbase_gpu_cacheclean(kbdev); -+ } -+ } -+#endif /* PLATFORM_POWER_DOWN_ONLY */ -+ -+ if (!backend->poweron_required) { -+#if !PLATFORM_POWER_DOWN_ONLY -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ WARN_ON(kbdev->l2_available_bitmap || -+ kbdev->shader_available_bitmap || -+ kbdev->tiler_available_bitmap); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+#endif /* !PLATFORM_POWER_DOWN_ONLY */ -+ -+ /* Consume any change-state events */ -+ kbase_timeline_pm_check_handle_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); -+ -+ /* Disable interrupts and turn the clock off */ -+ if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) { -+ /* -+ * Page/bus faults are pending, must drop locks to -+ * process. Interrupts are disabled so no more faults -+ * should be generated at this point. -+ */ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ kbase_flush_mmu_wqs(kbdev); -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+ /* Turn off clock now that fault have been handled. We -+ * dropped locks so poweron_required may have changed - -+ * power back on if this is the case.*/ -+ if (backend->poweron_required) -+ kbase_pm_clock_on(kbdev, false); -+ else -+ WARN_ON(!kbase_pm_clock_off(kbdev, -+ backend->poweroff_is_suspend)); -+ } -+ } -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ backend->poweroff_wait_in_progress = false; -+ if (backend->poweron_required) { -+ backend->poweron_required = false; -+ kbase_pm_update_cores_state_nolock(kbdev); -+ kbase_backend_slot_update(kbdev); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ wake_up(&kbdev->pm.backend.poweroff_wait); -+} -+ -+void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) -+{ -+ unsigned long flags; -+ -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (!kbdev->pm.backend.poweroff_wait_in_progress) { -+ /* Force all cores off */ -+ kbdev->pm.backend.desired_shader_state = 0; -+ kbdev->pm.backend.desired_tiler_state = 0; -+ -+ /* Force all cores to be unavailable, in the situation where -+ * transitions are in progress for some cores but not others, -+ * and kbase_pm_check_transitions_nolock can not immediately -+ * power off the cores */ -+ kbdev->shader_available_bitmap = 0; -+ kbdev->tiler_available_bitmap = 0; -+ kbdev->l2_available_bitmap = 0; -+ -+ kbdev->pm.backend.poweroff_wait_in_progress = true; -+ kbdev->pm.backend.poweroff_is_suspend = is_suspend; -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ /*Kick off wq here. Callers will have to wait*/ -+ queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, -+ &kbdev->pm.backend.gpu_poweroff_wait_work); -+ } else { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+} -+ -+static bool is_poweroff_in_progress(struct kbase_device *kbdev) -+{ -+ bool ret; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ ret = (kbdev->pm.backend.poweroff_wait_in_progress == false); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return ret; -+} -+ -+void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev) -+{ -+ wait_event_killable(kbdev->pm.backend.poweroff_wait, -+ is_poweroff_in_progress(kbdev)); -+} -+ -+int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, -+ unsigned int flags) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ unsigned long irq_flags; -+ int ret; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+ /* A suspend won't happen during startup/insmod */ -+ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); -+ -+ /* Power up the GPU, don't enable IRQs as we are not ready to receive -+ * them. */ -+ ret = kbase_pm_init_hw(kbdev, flags); -+ if (ret) { -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ return ret; -+ } -+ -+ kbasep_pm_init_core_use_bitmaps(kbdev); -+ -+ kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = -+ kbdev->pm.debug_core_mask[1] = -+ kbdev->pm.debug_core_mask[2] = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ -+ /* Pretend the GPU is active to prevent a power policy turning the GPU -+ * cores off */ -+ kbdev->pm.active_count = 1; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ irq_flags); -+ /* Ensure cycle counter is off */ -+ kbdev->pm.backend.gpu_cycle_counter_requests = 0; -+ spin_unlock_irqrestore( -+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ irq_flags); -+ -+ /* We are ready to receive IRQ's now as power policy is set up, so -+ * enable them now. */ -+#ifdef CONFIG_MALI_DEBUG -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags); -+ kbdev->pm.backend.driver_ready_for_irqs = true; -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags); -+#endif -+ kbase_pm_enable_interrupts(kbdev); -+ -+ /* Turn on the GPU and any cores needed by the policy */ -+ kbase_pm_do_poweron(kbdev, false); -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ /* Idle the GPU and/or cores, if the policy wants it to */ -+ kbase_pm_context_idle(kbdev); -+ -+ return 0; -+} -+ -+void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ mutex_lock(&kbdev->pm.lock); -+ kbase_pm_cancel_deferred_poweroff(kbdev); -+ kbase_pm_do_poweroff(kbdev, false); -+ mutex_unlock(&kbdev->pm.lock); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); -+ -+void kbase_hwaccess_pm_term(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0); -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0); -+ -+ /* Free any resources the policy allocated */ -+ kbase_pm_policy_term(kbdev); -+ kbase_pm_ca_term(kbdev); -+ -+ /* Shut down the metrics subsystem */ -+ kbasep_pm_metrics_term(kbdev); -+ -+ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); -+} -+ -+void kbase_pm_power_changed(struct kbase_device *kbdev) -+{ -+ bool cores_are_available; -+ unsigned long flags; -+ -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END); -+ -+ if (cores_are_available) { -+ /* Log timelining information that a change in state has -+ * completed */ -+ kbase_timeline_pm_handle_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); -+ -+ kbase_backend_slot_update(kbdev); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, -+ u64 new_core_mask_js0, u64 new_core_mask_js1, -+ u64 new_core_mask_js2) -+{ -+ kbdev->pm.debug_core_mask[0] = new_core_mask_js0; -+ kbdev->pm.debug_core_mask[1] = new_core_mask_js1; -+ kbdev->pm.debug_core_mask[2] = new_core_mask_js2; -+ kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | -+ new_core_mask_js2; -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+} -+ -+void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) -+{ -+ kbase_pm_update_active(kbdev); -+} -+ -+void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev) -+{ -+ kbase_pm_update_active(kbdev); -+} -+ -+void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ -+ /* Force power off the GPU and all cores (regardless of policy), only -+ * after the PM active count reaches zero (otherwise, we risk turning it -+ * off prematurely) */ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+ kbase_pm_cancel_deferred_poweroff(kbdev); -+ kbase_pm_do_poweroff(kbdev, true); -+ -+ kbase_backend_timer_suspend(kbdev); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ kbase_pm_wait_for_poweroff_complete(kbdev); -+} -+ -+void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+ kbdev->pm.suspending = false; -+ kbase_pm_do_poweron(kbdev, true); -+ -+ kbase_backend_timer_resume(kbdev); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+} -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c -new file mode 100755 -index 000000000..85890f1e8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c -@@ -0,0 +1,182 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Base kernel core availability APIs -+ */ -+ -+#include -+#include -+#include -+ -+static const struct kbase_pm_ca_policy *const policy_list[] = { -+ &kbase_pm_ca_fixed_policy_ops, -+#ifdef CONFIG_MALI_DEVFREQ -+ &kbase_pm_ca_devfreq_policy_ops, -+#endif -+#if !MALI_CUSTOMER_RELEASE -+ &kbase_pm_ca_random_policy_ops -+#endif -+}; -+ -+/** -+ * POLICY_COUNT - The number of policies available in the system. -+ * -+ * This is derived from the number of functions listed in policy_list. -+ */ -+#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) -+ -+int kbase_pm_ca_init(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ kbdev->pm.backend.ca_current_policy = policy_list[0]; -+ -+ kbdev->pm.backend.ca_current_policy->init(kbdev); -+ -+ return 0; -+} -+ -+void kbase_pm_ca_term(struct kbase_device *kbdev) -+{ -+ kbdev->pm.backend.ca_current_policy->term(kbdev); -+} -+ -+int kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **list) -+{ -+ if (!list) -+ return POLICY_COUNT; -+ -+ *list = policy_list; -+ -+ return POLICY_COUNT; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_ca_list_policies); -+ -+const struct kbase_pm_ca_policy -+*kbase_pm_ca_get_policy(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ return kbdev->pm.backend.ca_current_policy; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_ca_get_policy); -+ -+void kbase_pm_ca_set_policy(struct kbase_device *kbdev, -+ const struct kbase_pm_ca_policy *new_policy) -+{ -+ const struct kbase_pm_ca_policy *old_policy; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(new_policy != NULL); -+ -+ KBASE_TRACE_ADD(kbdev, PM_CA_SET_POLICY, NULL, NULL, 0u, -+ new_policy->id); -+ -+ /* During a policy change we pretend the GPU is active */ -+ /* A suspend won't happen here, because we're in a syscall from a -+ * userspace thread */ -+ kbase_pm_context_active(kbdev); -+ -+ mutex_lock(&kbdev->pm.lock); -+ -+ /* Remove the policy to prevent IRQ handlers from working on it */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ old_policy = kbdev->pm.backend.ca_current_policy; -+ kbdev->pm.backend.ca_current_policy = NULL; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (old_policy->term) -+ old_policy->term(kbdev); -+ -+ if (new_policy->init) -+ new_policy->init(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.ca_current_policy = new_policy; -+ -+ /* If any core power state changes were previously attempted, but -+ * couldn't be made because the policy was changing (current_policy was -+ * NULL), then re-try them here. */ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, -+ kbdev->shader_ready_bitmap, -+ kbdev->shader_transitioning_bitmap); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ -+ /* Now the policy change is finished, we release our fake context active -+ * reference */ -+ kbase_pm_context_idle(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_ca_set_policy); -+ -+u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* All cores must be enabled when instrumentation is in use */ -+ if (kbdev->pm.backend.instr_enabled) -+ return kbdev->gpu_props.props.raw_props.shader_present & -+ kbdev->pm.debug_core_mask_all; -+ -+ if (kbdev->pm.backend.ca_current_policy == NULL) -+ return kbdev->gpu_props.props.raw_props.shader_present & -+ kbdev->pm.debug_core_mask_all; -+ -+ return kbdev->pm.backend.ca_current_policy->get_core_mask(kbdev) & -+ kbdev->pm.debug_core_mask_all; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); -+ -+void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, -+ u64 cores_transitioning) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (kbdev->pm.backend.ca_current_policy != NULL) -+ kbdev->pm.backend.ca_current_policy->update_core_status(kbdev, -+ cores_ready, -+ cores_transitioning); -+} -+ -+void kbase_pm_ca_instr_enable(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.instr_enabled = true; -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_pm_ca_instr_disable(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ kbdev->pm.backend.instr_enabled = false; -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+} -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h -new file mode 100755 -index 000000000..ee9e751f2 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h -@@ -0,0 +1,92 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Base kernel core availability APIs -+ */ -+ -+#ifndef _KBASE_PM_CA_H_ -+#define _KBASE_PM_CA_H_ -+ -+/** -+ * kbase_pm_ca_init - Initialize core availability framework -+ * -+ * Must be called before calling any other core availability function -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: 0 if the core availability framework was successfully initialized, -+ * -errno otherwise -+ */ -+int kbase_pm_ca_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_ca_term - Terminate core availability framework -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_ca_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_ca_get_core_mask - Get currently available shaders core mask -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Returns a mask of the currently available shader cores. -+ * Calls into the core availability policy -+ * -+ * Return: The bit mask of available cores -+ */ -+u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_ca_update_core_status - Update core status -+ * -+ * @kbdev: The kbase device structure for the device (must be -+ * a valid pointer) -+ * @cores_ready: The bit mask of cores ready for job submission -+ * @cores_transitioning: The bit mask of cores that are transitioning power -+ * state -+ * -+ * Update core availability policy with current core power status -+ * -+ * Calls into the core availability policy -+ */ -+void kbase_pm_ca_update_core_status(struct kbase_device *kbdev, u64 cores_ready, -+ u64 cores_transitioning); -+ -+/** -+ * kbase_pm_ca_instr_enable - Enable override for instrumentation -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * This overrides the output of the core availability policy, ensuring that all -+ * cores are available -+ */ -+void kbase_pm_ca_instr_enable(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_ca_instr_disable - Disable override for instrumentation -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * This disables any previously enabled override, and resumes normal policy -+ * functionality -+ */ -+void kbase_pm_ca_instr_disable(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_PM_CA_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c -new file mode 100755 -index 000000000..66bf660cf ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.c -@@ -0,0 +1,129 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * A core availability policy implementing core mask selection from devfreq OPPs -+ * -+ */ -+ -+#include -+#include -+#include -+#include -+ -+void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) -+{ -+ struct kbasep_pm_ca_policy_devfreq *data = -+ &kbdev->pm.backend.ca_policy_data.devfreq; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ data->cores_desired = core_mask; -+ -+ /* Disable any cores that are now unwanted */ -+ data->cores_enabled &= data->cores_desired; -+ -+ kbdev->pm.backend.ca_in_transition = true; -+ -+ /* If there are no cores to be powered off then power on desired cores -+ */ -+ if (!(data->cores_used & ~data->cores_desired)) { -+ data->cores_enabled = data->cores_desired; -+ kbdev->pm.backend.ca_in_transition = false; -+ } -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX %llX\n", -+ data->cores_desired, data->cores_enabled); -+} -+ -+static void devfreq_init(struct kbase_device *kbdev) -+{ -+ struct kbasep_pm_ca_policy_devfreq *data = -+ &kbdev->pm.backend.ca_policy_data.devfreq; -+ -+ if (kbdev->current_core_mask) { -+ data->cores_enabled = kbdev->current_core_mask; -+ data->cores_desired = kbdev->current_core_mask; -+ } else { -+ data->cores_enabled = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ data->cores_desired = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ } -+ data->cores_used = 0; -+ kbdev->pm.backend.ca_in_transition = false; -+} -+ -+static void devfreq_term(struct kbase_device *kbdev) -+{ -+} -+ -+static u64 devfreq_get_core_mask(struct kbase_device *kbdev) -+{ -+ return kbdev->pm.backend.ca_policy_data.devfreq.cores_enabled; -+} -+ -+static void devfreq_update_core_status(struct kbase_device *kbdev, -+ u64 cores_ready, -+ u64 cores_transitioning) -+{ -+ struct kbasep_pm_ca_policy_devfreq *data = -+ &kbdev->pm.backend.ca_policy_data.devfreq; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ data->cores_used = cores_ready | cores_transitioning; -+ -+ /* If in desired state then clear transition flag */ -+ if (data->cores_enabled == data->cores_desired) -+ kbdev->pm.backend.ca_in_transition = false; -+ -+ /* If all undesired cores are now off then power on desired cores. -+ * The direct comparison against cores_enabled limits potential -+ * recursion to one level */ -+ if (!(data->cores_used & ~data->cores_desired) && -+ data->cores_enabled != data->cores_desired) { -+ data->cores_enabled = data->cores_desired; -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ kbdev->pm.backend.ca_in_transition = false; -+ } -+} -+ -+/* -+ * The struct kbase_pm_ca_policy structure for the devfreq core availability -+ * policy. -+ * -+ * This is the static structure that defines the devfreq core availability power -+ * policy's callback and name. -+ */ -+const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops = { -+ "devfreq", /* name */ -+ devfreq_init, /* init */ -+ devfreq_term, /* term */ -+ devfreq_get_core_mask, /* get_core_mask */ -+ devfreq_update_core_status, /* update_core_status */ -+ 0u, /* flags */ -+ KBASE_PM_CA_POLICY_ID_DEVFREQ, /* id */ -+}; -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h -new file mode 100755 -index 000000000..7ab3cd4d8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h -@@ -0,0 +1,55 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * A core availability policy for use with devfreq, where core masks are -+ * associated with OPPs. -+ */ -+ -+#ifndef MALI_KBASE_PM_CA_DEVFREQ_H -+#define MALI_KBASE_PM_CA_DEVFREQ_H -+ -+/** -+ * struct kbasep_pm_ca_policy_devfreq - Private structure for devfreq ca policy -+ * -+ * This contains data that is private to the devfreq core availability -+ * policy. -+ * -+ * @cores_desired: Cores that the policy wants to be available -+ * @cores_enabled: Cores that the policy is currently returning as available -+ * @cores_used: Cores currently powered or transitioning -+ */ -+struct kbasep_pm_ca_policy_devfreq { -+ u64 cores_desired; -+ u64 cores_enabled; -+ u64 cores_used; -+}; -+ -+extern const struct kbase_pm_ca_policy kbase_pm_ca_devfreq_policy_ops; -+ -+/** -+ * kbase_devfreq_set_core_mask - Set core mask for policy to use -+ * @kbdev: Device pointer -+ * @core_mask: New core mask -+ * -+ * The new core mask will have immediate effect if the GPU is powered, or will -+ * take effect when it is next powered on. -+ */ -+void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask); -+ -+#endif /* MALI_KBASE_PM_CA_DEVFREQ_H */ -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c -new file mode 100755 -index 000000000..864612d31 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.c -@@ -0,0 +1,65 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * A power policy implementing fixed core availability -+ */ -+ -+#include -+#include -+ -+static void fixed_init(struct kbase_device *kbdev) -+{ -+ kbdev->pm.backend.ca_in_transition = false; -+} -+ -+static void fixed_term(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+static u64 fixed_get_core_mask(struct kbase_device *kbdev) -+{ -+ return kbdev->gpu_props.props.raw_props.shader_present; -+} -+ -+static void fixed_update_core_status(struct kbase_device *kbdev, -+ u64 cores_ready, -+ u64 cores_transitioning) -+{ -+ CSTD_UNUSED(kbdev); -+ CSTD_UNUSED(cores_ready); -+ CSTD_UNUSED(cores_transitioning); -+} -+ -+/* -+ * The struct kbase_pm_policy structure for the fixed power policy. -+ * -+ * This is the static structure that defines the fixed power policy's callback -+ * and name. -+ */ -+const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops = { -+ "fixed", /* name */ -+ fixed_init, /* init */ -+ fixed_term, /* term */ -+ fixed_get_core_mask, /* get_core_mask */ -+ fixed_update_core_status, /* update_core_status */ -+ 0u, /* flags */ -+ KBASE_PM_CA_POLICY_ID_FIXED, /* id */ -+}; -+ -+KBASE_EXPORT_TEST_API(kbase_pm_ca_fixed_policy_ops); -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h -new file mode 100755 -index 000000000..a763155cb ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_fixed.h -@@ -0,0 +1,40 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * A power policy implementing fixed core availability -+ */ -+ -+#ifndef MALI_KBASE_PM_CA_FIXED_H -+#define MALI_KBASE_PM_CA_FIXED_H -+ -+/** -+ * struct kbasep_pm_ca_policy_fixed - Private structure for policy instance data -+ * -+ * @dummy: Dummy member - no state is needed -+ * -+ * This contains data that is private to the particular power policy that is -+ * active. -+ */ -+struct kbasep_pm_ca_policy_fixed { -+ int dummy; -+}; -+ -+extern const struct kbase_pm_ca_policy kbase_pm_ca_fixed_policy_ops; -+ -+#endif /* MALI_KBASE_PM_CA_FIXED_H */ -+ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c -new file mode 100755 -index 000000000..f891fa225 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c -@@ -0,0 +1,70 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * "Coarse Demand" power management policy -+ */ -+ -+#include -+#include -+ -+static u64 coarse_demand_get_core_mask(struct kbase_device *kbdev) -+{ -+ if (kbdev->pm.active_count == 0) -+ return 0; -+ -+ return kbdev->gpu_props.props.raw_props.shader_present; -+} -+ -+static bool coarse_demand_get_core_active(struct kbase_device *kbdev) -+{ -+ if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | -+ kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt -+ && !kbdev->tiler_inuse_cnt) -+ return false; -+ -+ return true; -+} -+ -+static void coarse_demand_init(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+static void coarse_demand_term(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+/* The struct kbase_pm_policy structure for the demand power policy. -+ * -+ * This is the static structure that defines the demand power policy's callback -+ * and name. -+ */ -+const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { -+ "coarse_demand", /* name */ -+ coarse_demand_init, /* init */ -+ coarse_demand_term, /* term */ -+ coarse_demand_get_core_mask, /* get_core_mask */ -+ coarse_demand_get_core_active, /* get_core_active */ -+ 0u, /* flags */ -+ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ -+}; -+ -+KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h -new file mode 100755 -index 000000000..749d305ee ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h -@@ -0,0 +1,64 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * "Coarse Demand" power management policy -+ */ -+ -+#ifndef MALI_KBASE_PM_COARSE_DEMAND_H -+#define MALI_KBASE_PM_COARSE_DEMAND_H -+ -+/** -+ * DOC: -+ * The "Coarse" demand power management policy has the following -+ * characteristics: -+ * - When KBase indicates that the GPU will be powered up, but we don't yet -+ * know which Job Chains are to be run: -+ * - All Shader Cores are powered up, regardless of whether or not they will -+ * be needed later. -+ * - When KBase indicates that a set of Shader Cores are needed to submit the -+ * currently queued Job Chains: -+ * - All Shader Cores are kept powered, regardless of whether or not they will -+ * be needed -+ * - When KBase indicates that the GPU need not be powered: -+ * - The Shader Cores are powered off, and the GPU itself is powered off too. -+ * -+ * @note: -+ * - KBase indicates the GPU will be powered up when it has a User Process that -+ * has just started to submit Job Chains. -+ * - KBase indicates the GPU need not be powered when all the Job Chains from -+ * User Processes have finished, and it is waiting for a User Process to -+ * submit some more Job Chains. -+ */ -+ -+/** -+ * struct kbasep_pm_policy_coarse_demand - Private structure for coarse demand -+ * policy -+ * -+ * This contains data that is private to the coarse demand power policy. -+ * -+ * @dummy: Dummy member - no state needed -+ */ -+struct kbasep_pm_policy_coarse_demand { -+ int dummy; -+}; -+ -+extern const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops; -+ -+#endif /* MALI_KBASE_PM_COARSE_DEMAND_H */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h -new file mode 100755 -index 000000000..352744ee6 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h -@@ -0,0 +1,519 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Backend-specific Power Manager definitions -+ */ -+ -+#ifndef _KBASE_PM_HWACCESS_DEFS_H_ -+#define _KBASE_PM_HWACCESS_DEFS_H_ -+ -+#include "mali_kbase_pm_ca_fixed.h" -+#include "mali_kbase_pm_ca_devfreq.h" -+#if !MALI_CUSTOMER_RELEASE -+#include "mali_kbase_pm_ca_random.h" -+#endif -+ -+#include "mali_kbase_pm_always_on.h" -+#include "mali_kbase_pm_coarse_demand.h" -+#include "mali_kbase_pm_demand.h" -+#if !MALI_CUSTOMER_RELEASE -+#include "mali_kbase_pm_demand_always_powered.h" -+#include "mali_kbase_pm_fast_start.h" -+#endif -+ -+/* Forward definition - see mali_kbase.h */ -+struct kbase_device; -+struct kbase_jd_atom; -+ -+/** -+ * enum kbase_pm_core_type - The types of core in a GPU. -+ * -+ * These enumerated values are used in calls to -+ * - kbase_pm_get_present_cores() -+ * - kbase_pm_get_active_cores() -+ * - kbase_pm_get_trans_cores() -+ * - kbase_pm_get_ready_cores(). -+ * -+ * They specify which type of core should be acted on. These values are set in -+ * a manner that allows core_type_to_reg() function to be simpler and more -+ * efficient. -+ * -+ * @KBASE_PM_CORE_L2: The L2 cache -+ * @KBASE_PM_CORE_SHADER: Shader cores -+ * @KBASE_PM_CORE_TILER: Tiler cores -+ * @KBASE_PM_CORE_STACK: Core stacks -+ */ -+enum kbase_pm_core_type { -+ KBASE_PM_CORE_L2 = L2_PRESENT_LO, -+ KBASE_PM_CORE_SHADER = SHADER_PRESENT_LO, -+ KBASE_PM_CORE_TILER = TILER_PRESENT_LO, -+ KBASE_PM_CORE_STACK = STACK_PRESENT_LO -+}; -+ -+/** -+ * struct kbasep_pm_metrics_data - Metrics data collected for use by the power -+ * management framework. -+ * -+ * @time_period_start: time at which busy/idle measurements started -+ * @time_busy: number of ns the GPU was busy executing jobs since the -+ * @time_period_start timestamp. -+ * @time_idle: number of ns since time_period_start the GPU was not executing -+ * jobs since the @time_period_start timestamp. -+ * @prev_busy: busy time in ns of previous time period. -+ * Updated when metrics are reset. -+ * @prev_idle: idle time in ns of previous time period -+ * Updated when metrics are reset. -+ * @gpu_active: true when the GPU is executing jobs. false when -+ * not. Updated when the job scheduler informs us a job in submitted -+ * or removed from a GPU slot. -+ * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that -+ * if two CL jobs were active for 400ns, this value would be updated -+ * with 800. -+ * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that -+ * if two GL jobs were active for 400ns, this value would be updated -+ * with 800. -+ * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. -+ * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. As -+ * GL jobs never run on slot 2 this slot is not recorded. -+ * @lock: spinlock protecting the kbasep_pm_metrics_data structure -+ * @timer: timer to regularly make DVFS decisions based on the power -+ * management metrics. -+ * @timer_active: boolean indicating @timer is running -+ * @platform_data: pointer to data controlled by platform specific code -+ * @kbdev: pointer to kbase device for which metrics are collected -+ * -+ */ -+struct kbasep_pm_metrics_data { -+ ktime_t time_period_start; -+ u32 time_busy; -+ u32 time_idle; -+ u32 prev_busy; -+ u32 prev_idle; -+ bool gpu_active; -+ u32 busy_cl[2]; -+ u32 busy_gl; -+ u32 active_cl_ctx[2]; -+ u32 active_gl_ctx[2]; /* GL jobs can only run on 2 of the 3 job slots */ -+ spinlock_t lock; -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ struct hrtimer timer; -+ bool timer_active; -+#endif -+ -+ void *platform_data; -+ struct kbase_device *kbdev; -+}; -+ -+union kbase_pm_policy_data { -+ struct kbasep_pm_policy_always_on always_on; -+ struct kbasep_pm_policy_coarse_demand coarse_demand; -+ struct kbasep_pm_policy_demand demand; -+#if !MALI_CUSTOMER_RELEASE -+ struct kbasep_pm_policy_demand_always_powered demand_always_powered; -+ struct kbasep_pm_policy_fast_start fast_start; -+#endif -+}; -+ -+union kbase_pm_ca_policy_data { -+ struct kbasep_pm_ca_policy_fixed fixed; -+ struct kbasep_pm_ca_policy_devfreq devfreq; -+#if !MALI_CUSTOMER_RELEASE -+ struct kbasep_pm_ca_policy_random random; -+#endif -+}; -+ -+/** -+ * struct kbase_pm_backend_data - Data stored per device for power management. -+ * -+ * This structure contains data for the power management framework. There is one -+ * instance of this structure per device in the system. -+ * -+ * @ca_current_policy: The policy that is currently actively controlling core -+ * availability. -+ * @pm_current_policy: The policy that is currently actively controlling the -+ * power state. -+ * @ca_policy_data: Private data for current CA policy -+ * @pm_policy_data: Private data for current PM policy -+ * @ca_in_transition: Flag indicating when core availability policy is -+ * transitioning cores. The core availability policy must -+ * set this when a change in core availability is occurring. -+ * power_change_lock must be held when accessing this. -+ * @reset_done: Flag when a reset is complete -+ * @reset_done_wait: Wait queue to wait for changes to @reset_done -+ * @l2_powered_wait: Wait queue for whether the l2 cache has been powered as -+ * requested -+ * @l2_powered: State indicating whether all the l2 caches are powered. -+ * Non-zero indicates they're *all* powered -+ * Zero indicates that some (or all) are not powered -+ * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter -+ * users -+ * @gpu_cycle_counter_requests_lock: Lock to protect @gpu_cycle_counter_requests -+ * @desired_shader_state: A bit mask identifying the shader cores that the -+ * power policy would like to be on. The current state -+ * of the cores may be different, but there should be -+ * transitions in progress that will eventually achieve -+ * this state (assuming that the policy doesn't change -+ * its mind in the mean time). -+ * @powering_on_shader_state: A bit mask indicating which shader cores are -+ * currently in a power-on transition -+ * @desired_tiler_state: A bit mask identifying the tiler cores that the power -+ * policy would like to be on. See @desired_shader_state -+ * @powering_on_tiler_state: A bit mask indicating which tiler core are -+ * currently in a power-on transition -+ * @powering_on_l2_state: A bit mask indicating which l2-caches are currently -+ * in a power-on transition -+ * @powering_on_stack_state: A bit mask indicating which core stacks are -+ * currently in a power-on transition -+ * @gpu_in_desired_state: This flag is set if the GPU is powered as requested -+ * by the desired_xxx_state variables -+ * @gpu_in_desired_state_wait: Wait queue set when @gpu_in_desired_state != 0 -+ * @gpu_powered: Set to true when the GPU is powered and register -+ * accesses are possible, false otherwise -+ * @instr_enabled: Set to true when instrumentation is enabled, -+ * false otherwise -+ * @cg1_disabled: Set if the policy wants to keep the second core group -+ * powered off -+ * @driver_ready_for_irqs: Debug state indicating whether sufficient -+ * initialization of the driver has occurred to handle -+ * IRQs -+ * @gpu_powered_lock: Spinlock that must be held when writing @gpu_powered or -+ * accessing @driver_ready_for_irqs -+ * @metrics: Structure to hold metrics for the GPU -+ * @gpu_poweroff_pending: number of poweroff timer ticks until the GPU is -+ * powered off -+ * @shader_poweroff_pending_time: number of poweroff timer ticks until shaders -+ * and/or timers are powered off -+ * @gpu_poweroff_timer: Timer for powering off GPU -+ * @gpu_poweroff_wq: Workqueue to power off GPU on when timer fires -+ * @gpu_poweroff_work: Workitem used on @gpu_poweroff_wq -+ * @shader_poweroff_pending: Bit mask of shaders to be powered off on next -+ * timer callback -+ * @tiler_poweroff_pending: Bit mask of tilers to be powered off on next timer -+ * callback -+ * @poweroff_timer_needed: true if the poweroff timer is currently required, -+ * false otherwise -+ * @poweroff_timer_running: true if the poweroff timer is currently running, -+ * false otherwise -+ * power_change_lock should be held when accessing, -+ * unless there is no way the timer can be running (eg -+ * hrtimer_cancel() was called immediately before) -+ * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. -+ * hwaccess_lock must be held when accessing -+ * @poweron_required: true if a GPU power on is required. Should only be set -+ * when poweroff_wait_in_progress is true, and therefore the -+ * GPU can not immediately be powered on. pm.lock must be -+ * held when accessing -+ * @poweroff_is_suspend: true if the GPU is being powered off due to a suspend -+ * request. pm.lock must be held when accessing -+ * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off -+ * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq -+ * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete -+ * @callback_power_on: Callback when the GPU needs to be turned on. See -+ * &struct kbase_pm_callback_conf -+ * @callback_power_off: Callback when the GPU may be turned off. See -+ * &struct kbase_pm_callback_conf -+ * @callback_power_suspend: Callback when a suspend occurs and the GPU needs to -+ * be turned off. See &struct kbase_pm_callback_conf -+ * @callback_power_resume: Callback when a resume occurs and the GPU needs to -+ * be turned on. See &struct kbase_pm_callback_conf -+ * @callback_power_runtime_on: Callback when the GPU needs to be turned on. See -+ * &struct kbase_pm_callback_conf -+ * @callback_power_runtime_off: Callback when the GPU may be turned off. See -+ * &struct kbase_pm_callback_conf -+ * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See -+ * &struct kbase_pm_callback_conf -+ * -+ * Note: -+ * During an IRQ, @ca_current_policy or @pm_current_policy can be NULL when the -+ * policy is being changed with kbase_pm_ca_set_policy() or -+ * kbase_pm_set_policy(). The change is protected under -+ * kbase_device.pm.power_change_lock. Direct access to this -+ * from IRQ context must therefore check for NULL. If NULL, then -+ * kbase_pm_ca_set_policy() or kbase_pm_set_policy() will re-issue the policy -+ * functions that would have been done under IRQ. -+ */ -+struct kbase_pm_backend_data { -+ const struct kbase_pm_ca_policy *ca_current_policy; -+ const struct kbase_pm_policy *pm_current_policy; -+ union kbase_pm_ca_policy_data ca_policy_data; -+ union kbase_pm_policy_data pm_policy_data; -+ bool ca_in_transition; -+ bool reset_done; -+ wait_queue_head_t reset_done_wait; -+ wait_queue_head_t l2_powered_wait; -+ int l2_powered; -+ int gpu_cycle_counter_requests; -+ spinlock_t gpu_cycle_counter_requests_lock; -+ -+ u64 desired_shader_state; -+ u64 powering_on_shader_state; -+ u64 desired_tiler_state; -+ u64 powering_on_tiler_state; -+ u64 powering_on_l2_state; -+#ifdef CONFIG_MALI_CORESTACK -+ u64 powering_on_stack_state; -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ bool gpu_in_desired_state; -+ wait_queue_head_t gpu_in_desired_state_wait; -+ -+ bool gpu_powered; -+ -+ bool instr_enabled; -+ -+ bool cg1_disabled; -+ -+#ifdef CONFIG_MALI_DEBUG -+ bool driver_ready_for_irqs; -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ spinlock_t gpu_powered_lock; -+ -+ -+ struct kbasep_pm_metrics_data metrics; -+ -+ int gpu_poweroff_pending; -+ int shader_poweroff_pending_time; -+ -+ struct hrtimer gpu_poweroff_timer; -+ struct workqueue_struct *gpu_poweroff_wq; -+ struct work_struct gpu_poweroff_work; -+ -+ u64 shader_poweroff_pending; -+ u64 tiler_poweroff_pending; -+ -+ bool poweroff_timer_needed; -+ bool poweroff_timer_running; -+ -+ bool poweroff_wait_in_progress; -+ bool poweron_required; -+ bool poweroff_is_suspend; -+ -+ struct workqueue_struct *gpu_poweroff_wait_wq; -+ struct work_struct gpu_poweroff_wait_work; -+ -+ wait_queue_head_t poweroff_wait; -+ -+ int (*callback_power_on)(struct kbase_device *kbdev); -+ void (*callback_power_off)(struct kbase_device *kbdev); -+ void (*callback_power_suspend)(struct kbase_device *kbdev); -+ void (*callback_power_resume)(struct kbase_device *kbdev); -+ int (*callback_power_runtime_on)(struct kbase_device *kbdev); -+ void (*callback_power_runtime_off)(struct kbase_device *kbdev); -+ int (*callback_power_runtime_idle)(struct kbase_device *kbdev); -+}; -+ -+ -+/* List of policy IDs */ -+enum kbase_pm_policy_id { -+ KBASE_PM_POLICY_ID_DEMAND = 1, -+ KBASE_PM_POLICY_ID_ALWAYS_ON, -+ KBASE_PM_POLICY_ID_COARSE_DEMAND, -+#if !MALI_CUSTOMER_RELEASE -+ KBASE_PM_POLICY_ID_DEMAND_ALWAYS_POWERED, -+ KBASE_PM_POLICY_ID_FAST_START -+#endif -+}; -+ -+typedef u32 kbase_pm_policy_flags; -+ -+/** -+ * struct kbase_pm_policy - Power policy structure. -+ * -+ * Each power policy exposes a (static) instance of this structure which -+ * contains function pointers to the policy's methods. -+ * -+ * @name: The name of this policy -+ * @init: Function called when the policy is selected -+ * @term: Function called when the policy is unselected -+ * @get_core_mask: Function called to get the current shader core mask -+ * @get_core_active: Function called to get the current overall GPU power -+ * state -+ * @flags: Field indicating flags for this policy -+ * @id: Field indicating an ID for this policy. This is not -+ * necessarily the same as its index in the list returned -+ * by kbase_pm_list_policies(). -+ * It is used purely for debugging. -+ */ -+struct kbase_pm_policy { -+ char *name; -+ -+ /** -+ * Function called when the policy is selected -+ * -+ * This should initialize the kbdev->pm.pm_policy_data structure. It -+ * should not attempt to make any changes to hardware state. -+ * -+ * It is undefined what state the cores are in when the function is -+ * called. -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ */ -+ void (*init)(struct kbase_device *kbdev); -+ -+ /** -+ * Function called when the policy is unselected. -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ */ -+ void (*term)(struct kbase_device *kbdev); -+ -+ /** -+ * Function called to get the current shader core mask -+ * -+ * The returned mask should meet or exceed (kbdev->shader_needed_bitmap -+ * | kbdev->shader_inuse_bitmap). -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ * -+ * Return: The mask of shader cores to be powered -+ */ -+ u64 (*get_core_mask)(struct kbase_device *kbdev); -+ -+ /** -+ * Function called to get the current overall GPU power state -+ * -+ * This function should consider the state of kbdev->pm.active_count. If -+ * this count is greater than 0 then there is at least one active -+ * context on the device and the GPU should be powered. If it is equal -+ * to 0 then there are no active contexts and the GPU could be powered -+ * off if desired. -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ * -+ * Return: true if the GPU should be powered, false otherwise -+ */ -+ bool (*get_core_active)(struct kbase_device *kbdev); -+ -+ kbase_pm_policy_flags flags; -+ enum kbase_pm_policy_id id; -+}; -+ -+ -+enum kbase_pm_ca_policy_id { -+ KBASE_PM_CA_POLICY_ID_FIXED = 1, -+ KBASE_PM_CA_POLICY_ID_DEVFREQ, -+ KBASE_PM_CA_POLICY_ID_RANDOM -+}; -+ -+typedef u32 kbase_pm_ca_policy_flags; -+ -+/** -+ * Maximum length of a CA policy names -+ */ -+#define KBASE_PM_CA_MAX_POLICY_NAME_LEN 15 -+ -+/** -+ * struct kbase_pm_ca_policy - Core availability policy structure. -+ * -+ * Each core availability policy exposes a (static) instance of this structure -+ * which contains function pointers to the policy's methods. -+ * -+ * @name: The name of this policy -+ * @init: Function called when the policy is selected -+ * @term: Function called when the policy is unselected -+ * @get_core_mask: Function called to get the current shader core -+ * availability mask -+ * @update_core_status: Function called to update the current core status -+ * @flags: Field indicating flags for this policy -+ * @id: Field indicating an ID for this policy. This is not -+ * necessarily the same as its index in the list returned -+ * by kbase_pm_list_policies(). -+ * It is used purely for debugging. -+ */ -+struct kbase_pm_ca_policy { -+ char name[KBASE_PM_CA_MAX_POLICY_NAME_LEN + 1]; -+ -+ /** -+ * Function called when the policy is selected -+ * -+ * This should initialize the kbdev->pm.ca_policy_data structure. It -+ * should not attempt to make any changes to hardware state. -+ * -+ * It is undefined what state the cores are in when the function is -+ * called. -+ * -+ * @kbdev The kbase device structure for the device (must be a -+ * valid pointer) -+ */ -+ void (*init)(struct kbase_device *kbdev); -+ -+ /** -+ * Function called when the policy is unselected. -+ * -+ * @kbdev The kbase device structure for the device (must be a -+ * valid pointer) -+ */ -+ void (*term)(struct kbase_device *kbdev); -+ -+ /** -+ * Function called to get the current shader core availability mask -+ * -+ * When a change in core availability is occurring, the policy must set -+ * kbdev->pm.ca_in_transition to true. This is to indicate that -+ * reporting changes in power state cannot be optimized out, even if -+ * kbdev->pm.desired_shader_state remains unchanged. This must be done -+ * by any functions internal to the Core Availability Policy that change -+ * the return value of kbase_pm_ca_policy::get_core_mask. -+ * -+ * @kbdev The kbase device structure for the device (must be a -+ * valid pointer) -+ * -+ * Return: The current core availability mask -+ */ -+ u64 (*get_core_mask)(struct kbase_device *kbdev); -+ -+ /** -+ * Function called to update the current core status -+ * -+ * If none of the cores in core group 0 are ready or transitioning, then -+ * the policy must ensure that the next call to get_core_mask does not -+ * return 0 for all cores in core group 0. It is an error to disable -+ * core group 0 through the core availability policy. -+ * -+ * When a change in core availability has finished, the policy must set -+ * kbdev->pm.ca_in_transition to false. This is to indicate that -+ * changes in power state can once again be optimized out when -+ * kbdev->pm.desired_shader_state is unchanged. -+ * -+ * @kbdev: The kbase device structure for the device -+ * (must be a valid pointer) -+ * @cores_ready: The mask of cores currently powered and -+ * ready to run jobs -+ * @cores_transitioning: The mask of cores currently transitioning -+ * power state -+ */ -+ void (*update_core_status)(struct kbase_device *kbdev, u64 cores_ready, -+ u64 cores_transitioning); -+ -+ kbase_pm_ca_policy_flags flags; -+ -+ /** -+ * Field indicating an ID for this policy. This is not necessarily the -+ * same as its index in the list returned by kbase_pm_list_policies(). -+ * It is used purely for debugging. -+ */ -+ enum kbase_pm_ca_policy_id id; -+}; -+ -+#endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c -new file mode 100755 -index 000000000..81322fd0d ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.c -@@ -0,0 +1,73 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * A simple demand based power management policy -+ */ -+ -+#include -+#include -+ -+static u64 demand_get_core_mask(struct kbase_device *kbdev) -+{ -+ u64 desired = kbdev->shader_needed_bitmap | kbdev->shader_inuse_bitmap; -+ -+ if (0 == kbdev->pm.active_count) -+ return 0; -+ -+ return desired; -+} -+ -+static bool demand_get_core_active(struct kbase_device *kbdev) -+{ -+ if (0 == kbdev->pm.active_count && !(kbdev->shader_needed_bitmap | -+ kbdev->shader_inuse_bitmap) && !kbdev->tiler_needed_cnt -+ && !kbdev->tiler_inuse_cnt) -+ return false; -+ -+ return true; -+} -+ -+static void demand_init(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+static void demand_term(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+/* -+ * The struct kbase_pm_policy structure for the demand power policy. -+ * -+ * This is the static structure that defines the demand power policy's callback -+ * and name. -+ */ -+const struct kbase_pm_policy kbase_pm_demand_policy_ops = { -+ "demand", /* name */ -+ demand_init, /* init */ -+ demand_term, /* term */ -+ demand_get_core_mask, /* get_core_mask */ -+ demand_get_core_active, /* get_core_active */ -+ 0u, /* flags */ -+ KBASE_PM_POLICY_ID_DEMAND, /* id */ -+}; -+ -+KBASE_EXPORT_TEST_API(kbase_pm_demand_policy_ops); -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h -new file mode 100755 -index 000000000..c0c84b6e9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_demand.h -@@ -0,0 +1,64 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * A simple demand based power management policy -+ */ -+ -+#ifndef MALI_KBASE_PM_DEMAND_H -+#define MALI_KBASE_PM_DEMAND_H -+ -+/** -+ * DOC: Demand power management policy -+ * -+ * The demand power management policy has the following characteristics: -+ * - When KBase indicates that the GPU will be powered up, but we don't yet -+ * know which Job Chains are to be run: -+ * - The Shader Cores are not powered up -+ * -+ * - When KBase indicates that a set of Shader Cores are needed to submit the -+ * currently queued Job Chains: -+ * - Only those Shader Cores are powered up -+ * -+ * - When KBase indicates that the GPU need not be powered: -+ * - The Shader Cores are powered off, and the GPU itself is powered off too. -+ * -+ * Note: -+ * - KBase indicates the GPU will be powered up when it has a User Process that -+ * has just started to submit Job Chains. -+ * -+ * - KBase indicates the GPU need not be powered when all the Job Chains from -+ * User Processes have finished, and it is waiting for a User Process to -+ * submit some more Job Chains. -+ */ -+ -+/** -+ * struct kbasep_pm_policy_demand - Private structure for policy instance data -+ * -+ * @dummy: No state is needed, a dummy variable -+ * -+ * This contains data that is private to the demand power policy. -+ */ -+struct kbasep_pm_policy_demand { -+ int dummy; -+}; -+ -+extern const struct kbase_pm_policy kbase_pm_demand_policy_ops; -+ -+#endif /* MALI_KBASE_PM_DEMAND_H */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c -new file mode 100755 -index 000000000..82727937c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c -@@ -0,0 +1,1713 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Base kernel Power Management hardware control -+ */ -+ -+// #define ENABLE_DEBUG_LOG -+#include "../../platform/rk/custom_log.h" -+ -+#include -+#include -+#include -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#if MALI_MOCK_TEST -+#define MOCKABLE(function) function##_original -+#else -+#define MOCKABLE(function) function -+#endif /* MALI_MOCK_TEST */ -+ -+/** -+ * enum kbasep_pm_action - Actions that can be performed on a core. -+ * -+ * This enumeration is private to the file. Its values are set to allow -+ * core_type_to_reg() function, which decodes this enumeration, to be simpler -+ * and more efficient. -+ * -+ * @ACTION_PRESENT: The cores that are present -+ * @ACTION_READY: The cores that are ready -+ * @ACTION_PWRON: Power on the cores specified -+ * @ACTION_PWROFF: Power off the cores specified -+ * @ACTION_PWRTRANS: The cores that are transitioning -+ * @ACTION_PWRACTIVE: The cores that are active -+ */ -+enum kbasep_pm_action { -+ ACTION_PRESENT = 0, -+ ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO), -+ ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO), -+ ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO), -+ ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO), -+ ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO) -+}; -+ -+/*---------------------------------------------------------------------------*/ -+ -+static bool is_action_of_powering_off_l2(enum kbase_pm_core_type core_type, -+ enum kbasep_pm_action active) -+{ -+ return (KBASE_PM_CORE_L2 == core_type) && (ACTION_PWROFF == active); -+} -+ -+static bool is_action_of_powering_off_shader(enum kbase_pm_core_type core_type, -+ enum kbasep_pm_action active) -+{ -+ return (KBASE_PM_CORE_SHADER == core_type) && (ACTION_PWROFF == active); -+} -+ -+static bool is_action_of_powering_off_tiler(enum kbase_pm_core_type core_type, -+ enum kbasep_pm_action active) -+{ -+ return (KBASE_PM_CORE_TILER == core_type) && (ACTION_PWROFF == active); -+} -+ -+static u64 kbase_pm_get_state( -+ struct kbase_device *kbdev, -+ enum kbase_pm_core_type core_type, -+ enum kbasep_pm_action action); -+ -+/** -+ * core_type_to_reg - Decode a core type and action to a register. -+ * -+ * Given a core type (defined by kbase_pm_core_type) and an action (defined -+ * by kbasep_pm_action) this function will return the register offset that -+ * will perform the action on the core type. The register returned is the _LO -+ * register and an offset must be applied to use the _HI register. -+ * -+ * @core_type: The type of core -+ * @action: The type of action -+ * -+ * Return: The register offset of the _LO register that performs an action of -+ * type @action on a core of type @core_type. -+ */ -+static u32 core_type_to_reg(enum kbase_pm_core_type core_type, -+ enum kbasep_pm_action action) -+{ -+#ifdef CONFIG_MALI_CORESTACK -+ if (core_type == KBASE_PM_CORE_STACK) { -+ switch (action) { -+ case ACTION_PRESENT: -+ return STACK_PRESENT_LO; -+ case ACTION_READY: -+ return STACK_READY_LO; -+ case ACTION_PWRON: -+ return STACK_PWRON_LO; -+ case ACTION_PWROFF: -+ return STACK_PWROFF_LO; -+ case ACTION_PWRTRANS: -+ return STACK_PWRTRANS_LO; -+ default: -+ BUG(); -+ } -+ } -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ return (u32)core_type + (u32)action; -+} -+ -+#ifdef CONFIG_ARM64 -+static void mali_cci_flush_l2(struct kbase_device *kbdev) -+{ -+ const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; -+ u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; -+ u32 raw; -+ -+ /* -+ * Note that we don't take the cache flush mutex here since -+ * we expect to be the last user of the L2, all other L2 users -+ * would have dropped their references, to initiate L2 power -+ * down, L2 power down being the only valid place for this -+ * to be called from. -+ */ -+ -+ kbase_reg_write(kbdev, -+ GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAN_INV_CACHES, -+ NULL); -+ -+ raw = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), -+ NULL); -+ -+ /* Wait for cache flush to complete before continuing, exit on -+ * gpu resets or loop expiry. */ -+ while (((raw & mask) == 0) && --loops) { -+ raw = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), -+ NULL); -+ } -+} -+#endif -+ -+/** -+ * kbase_pm_invoke - Invokes an action on a core set -+ * -+ * This function performs the action given by @action on a set of cores of a -+ * type given by @core_type. It is a static function used by -+ * kbase_pm_transition_core_type() -+ * -+ * @kbdev: The kbase device structure of the device -+ * @core_type: The type of core that the action should be performed on -+ * @cores: A bit mask of cores to perform the action on (low 32 bits) -+ * @action: The action to perform on the cores -+ */ -+static void kbase_pm_invoke(struct kbase_device *kbdev, -+ enum kbase_pm_core_type core_type, -+ u64 cores, -+ enum kbasep_pm_action action) -+{ -+ u32 reg; -+ u32 lo = cores & 0xFFFFFFFF; -+ u32 hi = (cores >> 32) & 0xFFFFFFFF; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /*-------------------------------------------------------*/ -+ -+ if ( is_action_of_powering_off_l2(core_type, action) ) { -+ D("not to power off l2 actually."); -+ return; -+ } -+ if ( is_action_of_powering_off_shader(core_type, action) ) { -+ D("not to power off shader actually. cores_lo : 0x%x, hi : 0x%x.", -+ lo, -+ hi); -+ return; -+ } -+ if ( is_action_of_powering_off_tiler(core_type, action) ) { -+ D("not to power off tiler actually."); -+ return; -+ } -+ -+ /*-------------------------------------------------------*/ -+ -+ reg = core_type_to_reg(core_type, action); -+ -+ KBASE_DEBUG_ASSERT(reg); -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ if (cores) { -+ if (action == ACTION_PWRON) -+ kbase_trace_mali_pm_power_on(core_type, cores); -+ else if (action == ACTION_PWROFF) -+ kbase_trace_mali_pm_power_off(core_type, cores); -+ } -+#endif -+ -+ if (cores) { -+ u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); -+ -+ if (action == ACTION_PWRON) -+ state |= cores; -+ else if (action == ACTION_PWROFF) -+ state &= ~cores; -+ KBASE_TLSTREAM_AUX_PM_STATE(core_type, state); -+ } -+ -+ /* Tracing */ -+ if (cores) { -+ if (action == ACTION_PWRON) -+ switch (core_type) { -+ case KBASE_PM_CORE_SHADER: -+ KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, -+ lo); -+ break; -+ case KBASE_PM_CORE_TILER: -+ KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, -+ NULL, 0u, lo); -+ break; -+ case KBASE_PM_CORE_L2: -+ KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, -+ 0u, lo); -+ break; -+ default: -+ break; -+ } -+ else if (action == ACTION_PWROFF) -+ switch (core_type) { -+ case KBASE_PM_CORE_SHADER: -+ KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, -+ 0u, lo); -+ break; -+ case KBASE_PM_CORE_TILER: -+ KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, -+ NULL, 0u, lo); -+ break; -+ case KBASE_PM_CORE_L2: -+ KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, -+ 0u, lo); -+ /* disable snoops before L2 is turned off */ -+ kbase_pm_cache_snoop_disable(kbdev); -+ break; -+ default: -+ break; -+ } -+ } -+ -+ if (lo != 0) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL); -+ -+ if (hi != 0) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL); -+} -+ -+/** -+ * kbase_pm_get_state - Get information about a core set -+ * -+ * This function gets information (chosen by @action) about a set of cores of -+ * a type given by @core_type. It is a static function used by -+ * kbase_pm_get_active_cores(), kbase_pm_get_trans_cores() and -+ * kbase_pm_get_ready_cores(). -+ * -+ * @kbdev: The kbase device structure of the device -+ * @core_type: The type of core that the should be queried -+ * @action: The property of the cores to query -+ * -+ * Return: A bit mask specifying the state of the cores -+ */ -+static u64 kbase_pm_get_state(struct kbase_device *kbdev, -+ enum kbase_pm_core_type core_type, -+ enum kbasep_pm_action action) -+{ -+ u32 reg; -+ u32 lo, hi; -+ -+ reg = core_type_to_reg(core_type, action); -+ -+ KBASE_DEBUG_ASSERT(reg); -+ -+ lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL); -+ hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL); -+ -+ return (((u64) hi) << 32) | ((u64) lo); -+} -+ -+void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev) -+{ -+ kbdev->shader_inuse_bitmap = 0; -+ kbdev->shader_needed_bitmap = 0; -+ kbdev->shader_available_bitmap = 0; -+ kbdev->tiler_available_bitmap = 0; -+ kbdev->l2_users_count = 0; -+ kbdev->l2_available_bitmap = 0; -+ kbdev->tiler_needed_cnt = 0; -+ kbdev->tiler_inuse_cnt = 0; -+ -+ memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt)); -+} -+ -+/** -+ * kbase_pm_get_present_cores - Get the cores that are present -+ * -+ * @kbdev: Kbase device -+ * @type: The type of cores to query -+ * -+ * Return: Bitmask of the cores that are present -+ */ -+u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ switch (type) { -+ case KBASE_PM_CORE_L2: -+ return kbdev->gpu_props.props.raw_props.l2_present; -+ case KBASE_PM_CORE_SHADER: -+ return kbdev->gpu_props.props.raw_props.shader_present; -+ case KBASE_PM_CORE_TILER: -+ return kbdev->gpu_props.props.raw_props.tiler_present; -+#ifdef CONFIG_MALI_CORESTACK -+ case KBASE_PM_CORE_STACK: -+ return kbdev->gpu_props.props.raw_props.stack_present; -+#endif /* CONFIG_MALI_CORESTACK */ -+ default: -+ break; -+ } -+ KBASE_DEBUG_ASSERT(0); -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores); -+ -+/** -+ * kbase_pm_get_active_cores - Get the cores that are "active" -+ * (busy processing work) -+ * -+ * @kbdev: Kbase device -+ * @type: The type of cores to query -+ * -+ * Return: Bitmask of cores that are active -+ */ -+u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type) -+{ -+ return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores); -+ -+/** -+ * kbase_pm_get_trans_cores - Get the cores that are transitioning between -+ * power states -+ * -+ * @kbdev: Kbase device -+ * @type: The type of cores to query -+ * -+ * Return: Bitmask of cores that are transitioning -+ */ -+u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type) -+{ -+ return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores); -+ -+/** -+ * kbase_pm_get_ready_cores - Get the cores that are powered on -+ * -+ * @kbdev: Kbase device -+ * @type: The type of cores to query -+ * -+ * Return: Bitmask of cores that are ready (powered on) -+ */ -+u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type) -+{ -+ u64 result; -+ -+ result = kbase_pm_get_state(kbdev, type, ACTION_READY); -+ -+ switch (type) { -+ case KBASE_PM_CORE_SHADER: -+ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, -+ (u32) result); -+ break; -+ case KBASE_PM_CORE_TILER: -+ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, -+ (u32) result); -+ break; -+ case KBASE_PM_CORE_L2: -+ KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, -+ (u32) result); -+ break; -+ default: -+ break; -+ } -+ -+ return result; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores); -+ -+/** -+ * kbase_pm_transition_core_type - Perform power transitions for a particular -+ * core type. -+ * -+ * This function will perform any available power transitions to make the actual -+ * hardware state closer to the desired state. If a core is currently -+ * transitioning then changes to the power state of that call cannot be made -+ * until the transition has finished. Cores which are not present in the -+ * hardware are ignored if they are specified in the desired_state bitmask, -+ * however the return value will always be 0 in this case. -+ * -+ * @kbdev: The kbase device -+ * @type: The core type to perform transitions for -+ * @desired_state: A bit mask of the desired state of the cores -+ * @in_use: A bit mask of the cores that are currently running -+ * jobs. These cores have to be kept powered up because -+ * there are jobs running (or about to run) on them. -+ * @available: Receives a bit mask of the cores that the job -+ * scheduler can use to submit jobs to. May be NULL if -+ * this is not needed. -+ * @powering_on: Bit mask to update with cores that are -+ * transitioning to a power-on state. -+ * -+ * Return: true if the desired state has been reached, false otherwise -+ */ -+static bool kbase_pm_transition_core_type(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type, -+ u64 desired_state, -+ u64 in_use, -+ u64 * const available, -+ u64 *powering_on) -+{ -+ u64 present; -+ u64 ready; -+ u64 trans; -+ u64 powerup; -+ u64 powerdown; -+ u64 powering_on_trans; -+ u64 desired_state_in_use; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* Get current state */ -+ present = kbase_pm_get_present_cores(kbdev, type); -+ trans = kbase_pm_get_trans_cores(kbdev, type); -+ ready = kbase_pm_get_ready_cores(kbdev, type); -+ /* mask off ready from trans in case transitions finished between the -+ * register reads */ -+ trans &= ~ready; -+ -+ if (trans) /* Do not progress if any cores are transitioning */ -+ return false; -+ -+ powering_on_trans = trans & *powering_on; -+ *powering_on = powering_on_trans; -+ -+ if (available != NULL) -+ *available = (ready | powering_on_trans) & desired_state; -+ -+ /* Update desired state to include the in-use cores. These have to be -+ * kept powered up because there are jobs running or about to run on -+ * these cores -+ */ -+ desired_state_in_use = desired_state | in_use; -+ -+ /* Update state of whether l2 caches are powered */ -+ if (type == KBASE_PM_CORE_L2) { -+ if ((ready == present) && (desired_state_in_use == ready) && -+ (trans == 0)) { -+ /* All are ready, none will be turned off, and none are -+ * transitioning */ -+ kbdev->pm.backend.l2_powered = 1; -+ /* -+ * Ensure snoops are enabled after L2 is powered up, -+ * note that kbase keeps track of the snoop state, so -+ * safe to repeatedly call. -+ */ -+ kbase_pm_cache_snoop_enable(kbdev); -+ if (kbdev->l2_users_count > 0) { -+ /* Notify any registered l2 cache users -+ * (optimized out when no users waiting) */ -+ wake_up(&kbdev->pm.backend.l2_powered_wait); -+ } -+ } else -+ kbdev->pm.backend.l2_powered = 0; -+ } -+ -+ if (desired_state == ready && (trans == 0)) -+ return true; -+ -+ /* Restrict the cores to those that are actually present */ -+ powerup = desired_state_in_use & present; -+ powerdown = (~desired_state_in_use) & present; -+ -+ /* Restrict to cores that are not already in the desired state */ -+ powerup &= ~ready; -+ powerdown &= ready; -+ -+ /* Don't transition any cores that are already transitioning, except for -+ * Mali cores that support the following case: -+ * -+ * If the SHADER_PWRON or TILER_PWRON registers are written to turn on -+ * a core that is currently transitioning to power off, then this is -+ * remembered and the shader core is automatically powered up again once -+ * the original transition completes. Once the automatic power on is -+ * complete any job scheduled on the shader core should start. -+ */ -+ powerdown &= ~trans; -+ -+ if (kbase_hw_has_feature(kbdev, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS)) -+ if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type) -+ trans = powering_on_trans; /* for exception cases, only -+ * mask off cores in power on -+ * transitions */ -+ -+ powerup &= ~trans; -+ -+ /* Perform transitions if any */ -+ kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON); -+#if !PLATFORM_POWER_DOWN_ONLY -+ kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF); -+#endif -+ -+ /* Recalculate cores transitioning on, and re-evaluate our state */ -+ powering_on_trans |= powerup; -+ *powering_on = powering_on_trans; -+ if (available != NULL) -+ *available = (ready | powering_on_trans) & desired_state; -+ -+ return false; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type); -+ -+/** -+ * get_desired_cache_status - Determine which caches should be on for a -+ * particular core state -+ * -+ * This function takes a bit mask of the present caches and the cores (or -+ * caches) that are attached to the caches that will be powered. It then -+ * computes which caches should be turned on to allow the cores requested to be -+ * powered up. -+ * -+ * @present: The bit mask of present caches -+ * @cores_powered: A bit mask of cores (or L2 caches) that are desired to -+ * be powered -+ * @tilers_powered: The bit mask of tilers that are desired to be powered -+ * -+ * Return: A bit mask of the caches that should be turned on -+ */ -+static u64 get_desired_cache_status(u64 present, u64 cores_powered, -+ u64 tilers_powered) -+{ -+ u64 desired = 0; -+ -+ while (present) { -+ /* Find out which is the highest set bit */ -+ u64 bit = fls64(present) - 1; -+ u64 bit_mask = 1ull << bit; -+ /* Create a mask which has all bits from 'bit' upwards set */ -+ -+ u64 mask = ~(bit_mask - 1); -+ -+ /* If there are any cores powered at this bit or above (that -+ * haven't previously been processed) then we need this core on -+ */ -+ if (cores_powered & mask) -+ desired |= bit_mask; -+ -+ /* Remove bits from cores_powered and present */ -+ cores_powered &= ~mask; -+ present &= ~bit_mask; -+ } -+ -+ /* Power up the required L2(s) for the tiler */ -+ if (tilers_powered) -+ desired |= 1; -+ -+ return desired; -+} -+ -+KBASE_EXPORT_TEST_API(get_desired_cache_status); -+ -+#ifdef CONFIG_MALI_CORESTACK -+u64 kbase_pm_core_stack_mask(u64 cores) -+{ -+ u64 stack_mask = 0; -+ size_t const MAX_CORE_ID = 31; -+ size_t const NUM_CORES_PER_STACK = 4; -+ size_t i; -+ -+ for (i = 0; i <= MAX_CORE_ID; ++i) { -+ if (test_bit(i, (unsigned long *)&cores)) { -+ /* Every core which ID >= 16 is filled to stacks 4-7 -+ * instead of 0-3 */ -+ size_t const stack_num = (i > 16) ? -+ (i % NUM_CORES_PER_STACK) + 4 : -+ (i % NUM_CORES_PER_STACK); -+ set_bit(stack_num, (unsigned long *)&stack_mask); -+ } -+ } -+ -+ return stack_mask; -+} -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+bool -+MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev) -+{ -+ bool cores_are_available = false; -+ bool in_desired_state = true; -+ u64 desired_l2_state; -+#ifdef CONFIG_MALI_CORESTACK -+ u64 desired_stack_state; -+ u64 stacks_powered; -+#endif /* CONFIG_MALI_CORESTACK */ -+ u64 cores_powered; -+ u64 tilers_powered; -+ u64 tiler_available_bitmap; -+ u64 tiler_transitioning_bitmap; -+ u64 shader_available_bitmap; -+ u64 shader_ready_bitmap; -+ u64 shader_transitioning_bitmap; -+ u64 l2_available_bitmap; -+ u64 prev_l2_available_bitmap; -+ u64 l2_inuse_bitmap; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ spin_lock(&kbdev->pm.backend.gpu_powered_lock); -+ if (kbdev->pm.backend.gpu_powered == false) { -+ spin_unlock(&kbdev->pm.backend.gpu_powered_lock); -+ if (kbdev->pm.backend.desired_shader_state == 0 && -+ kbdev->pm.backend.desired_tiler_state == 0) -+ return true; -+ return false; -+ } -+ -+ /* Trace that a change-state is being requested, and that it took -+ * (effectively) no time to start it. This is useful for counting how -+ * many state changes occurred, in a way that's backwards-compatible -+ * with processing the trace data */ -+ kbase_timeline_pm_send_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); -+ kbase_timeline_pm_handle_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE); -+ -+ /* If any cores are already powered then, we must keep the caches on */ -+ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, -+ KBASE_PM_CORE_SHADER); -+ cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); -+ cores_powered |= kbdev->pm.backend.desired_shader_state; -+ -+#ifdef CONFIG_MALI_CORESTACK -+ /* Work out which core stacks want to be powered */ -+ desired_stack_state = kbase_pm_core_stack_mask(cores_powered); -+ stacks_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK) | -+ desired_stack_state; -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ /* Work out which tilers want to be powered */ -+ tiler_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, -+ KBASE_PM_CORE_TILER); -+ tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER); -+ tilers_powered |= kbdev->pm.backend.desired_tiler_state; -+ -+ /* If there are l2 cache users registered, keep all l2s powered even if -+ * all other cores are off. */ -+ if (kbdev->l2_users_count > 0) -+ cores_powered |= kbdev->gpu_props.props.raw_props.l2_present; -+ -+ desired_l2_state = get_desired_cache_status( -+ kbdev->gpu_props.props.raw_props.l2_present, -+ cores_powered, tilers_powered); -+ -+ l2_inuse_bitmap = get_desired_cache_status( -+ kbdev->gpu_props.props.raw_props.l2_present, -+ cores_powered | shader_transitioning_bitmap, -+ tilers_powered | tiler_transitioning_bitmap); -+ -+#ifdef CONFIG_MALI_CORESTACK -+ if (stacks_powered) -+ desired_l2_state |= 1; -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ /* If any l2 cache is on, then enable l2 #0, for use by job manager */ -+ if (0 != desired_l2_state) -+ desired_l2_state |= 1; -+ -+ prev_l2_available_bitmap = kbdev->l2_available_bitmap; -+ in_desired_state &= kbase_pm_transition_core_type(kbdev, -+ KBASE_PM_CORE_L2, desired_l2_state, l2_inuse_bitmap, -+ &l2_available_bitmap, -+ &kbdev->pm.backend.powering_on_l2_state); -+ -+ if (kbdev->l2_available_bitmap != l2_available_bitmap) -+ KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap); -+ -+ kbdev->l2_available_bitmap = l2_available_bitmap; -+ -+ -+#ifdef CONFIG_MALI_CORESTACK -+ if (in_desired_state) { -+ in_desired_state &= kbase_pm_transition_core_type(kbdev, -+ KBASE_PM_CORE_STACK, desired_stack_state, 0, -+ &kbdev->stack_available_bitmap, -+ &kbdev->pm.backend.powering_on_stack_state); -+ } -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ if (in_desired_state) { -+ in_desired_state &= kbase_pm_transition_core_type(kbdev, -+ KBASE_PM_CORE_TILER, -+ kbdev->pm.backend.desired_tiler_state, -+ 0, &tiler_available_bitmap, -+ &kbdev->pm.backend.powering_on_tiler_state); -+ in_desired_state &= kbase_pm_transition_core_type(kbdev, -+ KBASE_PM_CORE_SHADER, -+ kbdev->pm.backend.desired_shader_state, -+ kbdev->shader_inuse_bitmap, -+ &shader_available_bitmap, -+ &kbdev->pm.backend.powering_on_shader_state); -+ -+ if (kbdev->shader_available_bitmap != shader_available_bitmap) { -+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, -+ NULL, 0u, -+ (u32) shader_available_bitmap); -+ KBASE_TIMELINE_POWER_SHADER(kbdev, -+ shader_available_bitmap); -+ } -+ -+ kbdev->shader_available_bitmap = shader_available_bitmap; -+ -+ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) { -+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, -+ NULL, NULL, 0u, -+ (u32) tiler_available_bitmap); -+ KBASE_TIMELINE_POWER_TILER(kbdev, -+ tiler_available_bitmap); -+ } -+ -+ kbdev->tiler_available_bitmap = tiler_available_bitmap; -+ -+ } else if ((l2_available_bitmap & -+ kbdev->gpu_props.props.raw_props.tiler_present) != -+ kbdev->gpu_props.props.raw_props.tiler_present) { -+ tiler_available_bitmap = 0; -+ -+ if (kbdev->tiler_available_bitmap != tiler_available_bitmap) -+ KBASE_TIMELINE_POWER_TILER(kbdev, -+ tiler_available_bitmap); -+ -+ kbdev->tiler_available_bitmap = tiler_available_bitmap; -+ } -+ -+ /* State updated for slow-path waiters */ -+ kbdev->pm.backend.gpu_in_desired_state = in_desired_state; -+ -+ shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, -+ KBASE_PM_CORE_SHADER); -+ shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, -+ KBASE_PM_CORE_SHADER); -+ -+ /* Determine whether the cores are now available (even if the set of -+ * available cores is empty). Note that they can be available even if -+ * we've not finished transitioning to the desired state */ -+ if ((kbdev->shader_available_bitmap & -+ kbdev->pm.backend.desired_shader_state) -+ == kbdev->pm.backend.desired_shader_state && -+ (kbdev->tiler_available_bitmap & -+ kbdev->pm.backend.desired_tiler_state) -+ == kbdev->pm.backend.desired_tiler_state) { -+ cores_are_available = true; -+ -+ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, -+ (u32)(kbdev->shader_available_bitmap & -+ kbdev->pm.backend.desired_shader_state)); -+ KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, -+ (u32)(kbdev->tiler_available_bitmap & -+ kbdev->pm.backend.desired_tiler_state)); -+ -+ /* Log timelining information about handling events that power -+ * up cores, to match up either with immediate submission either -+ * because cores already available, or from PM IRQ */ -+ if (!in_desired_state) -+ kbase_timeline_pm_send_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); -+ } -+ -+ if (in_desired_state) { -+ KBASE_DEBUG_ASSERT(cores_are_available); -+ -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, -+ kbase_pm_get_ready_cores(kbdev, -+ KBASE_PM_CORE_L2)); -+ kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, -+ kbase_pm_get_ready_cores(kbdev, -+ KBASE_PM_CORE_SHADER)); -+ kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, -+ kbase_pm_get_ready_cores(kbdev, -+ KBASE_PM_CORE_TILER)); -+#ifdef CONFIG_MALI_CORESTACK -+ kbase_trace_mali_pm_status(KBASE_PM_CORE_STACK, -+ kbase_pm_get_ready_cores(kbdev, -+ KBASE_PM_CORE_STACK)); -+#endif /* CONFIG_MALI_CORESTACK */ -+#endif -+ -+ KBASE_TLSTREAM_AUX_PM_STATE( -+ KBASE_PM_CORE_L2, -+ kbase_pm_get_ready_cores( -+ kbdev, KBASE_PM_CORE_L2)); -+ KBASE_TLSTREAM_AUX_PM_STATE( -+ KBASE_PM_CORE_SHADER, -+ kbase_pm_get_ready_cores( -+ kbdev, KBASE_PM_CORE_SHADER)); -+ KBASE_TLSTREAM_AUX_PM_STATE( -+ KBASE_PM_CORE_TILER, -+ kbase_pm_get_ready_cores( -+ kbdev, -+ KBASE_PM_CORE_TILER)); -+#ifdef CONFIG_MALI_CORESTACK -+ KBASE_TLSTREAM_AUX_PM_STATE( -+ KBASE_PM_CORE_STACK, -+ kbase_pm_get_ready_cores( -+ kbdev, -+ KBASE_PM_CORE_STACK)); -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, -+ kbdev->pm.backend.gpu_in_desired_state, -+ (u32)kbdev->pm.backend.desired_shader_state); -+ KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, -+ (u32)kbdev->pm.backend.desired_tiler_state); -+ -+ /* Log timelining information for synchronous waiters */ -+ kbase_timeline_pm_send_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); -+ /* Wake slow-path waiters. Job scheduler does not use this. */ -+ KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0); -+ -+ wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); -+ } -+ -+ spin_unlock(&kbdev->pm.backend.gpu_powered_lock); -+ -+ /* kbase_pm_ca_update_core_status can cause one-level recursion into -+ * this function, so it must only be called once all changes to kbdev -+ * have been committed, and after the gpu_powered_lock has been -+ * dropped. */ -+ if (kbdev->shader_ready_bitmap != shader_ready_bitmap || -+ kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) { -+ kbdev->shader_ready_bitmap = shader_ready_bitmap; -+ kbdev->shader_transitioning_bitmap = -+ shader_transitioning_bitmap; -+ -+ kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, -+ shader_transitioning_bitmap); -+ } -+ -+ /* The core availability policy is not allowed to keep core group 0 -+ * turned off (unless it was changing the l2 power state) */ -+ if (!((shader_ready_bitmap | shader_transitioning_bitmap) & -+ kbdev->gpu_props.props.coherency_info.group[0].core_mask) && -+ (prev_l2_available_bitmap == desired_l2_state) && -+ !(kbase_pm_ca_get_core_mask(kbdev) & -+ kbdev->gpu_props.props.coherency_info.group[0].core_mask)) -+ BUG(); -+ -+ /* The core availability policy is allowed to keep core group 1 off, -+ * but all jobs specifically targeting CG1 must fail */ -+ if (!((shader_ready_bitmap | shader_transitioning_bitmap) & -+ kbdev->gpu_props.props.coherency_info.group[1].core_mask) && -+ !(kbase_pm_ca_get_core_mask(kbdev) & -+ kbdev->gpu_props.props.coherency_info.group[1].core_mask)) -+ kbdev->pm.backend.cg1_disabled = true; -+ else -+ kbdev->pm.backend.cg1_disabled = false; -+ -+ return cores_are_available; -+} -+KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock); -+ -+/* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has -+ * aborted due to a fatal signal. If the time spent waiting has exceeded this -+ * threshold then there is most likely a hardware issue. */ -+#define PM_TIMEOUT (5*HZ) /* 5s */ -+ -+void kbase_pm_check_transitions_sync(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ unsigned long timeout; -+ bool cores_are_available; -+ int ret; -+ -+ /* Force the transition to be checked and reported - the cores may be -+ * 'available' (for job submission) but not fully powered up. */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); -+ -+ /* Don't need 'cores_are_available', because we don't return anything */ -+ CSTD_UNUSED(cores_are_available); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ timeout = jiffies + PM_TIMEOUT; -+ -+ /* Wait for cores */ -+ ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, -+ kbdev->pm.backend.gpu_in_desired_state); -+ -+ if (ret < 0 && time_after(jiffies, timeout)) { -+ dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); -+ dev_err(kbdev->dev, "Desired state :\n"); -+ dev_err(kbdev->dev, "\tShader=%016llx\n", -+ kbdev->pm.backend.desired_shader_state); -+ dev_err(kbdev->dev, "\tTiler =%016llx\n", -+ kbdev->pm.backend.desired_tiler_state); -+ dev_err(kbdev->dev, "Current state :\n"); -+ dev_err(kbdev->dev, "\tShader=%08x%08x\n", -+ kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_READY_HI), NULL), -+ kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_READY_LO), -+ NULL)); -+ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", -+ kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TILER_READY_HI), NULL), -+ kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TILER_READY_LO), NULL)); -+ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", -+ kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_READY_HI), NULL), -+ kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_READY_LO), NULL)); -+ dev_err(kbdev->dev, "Cores transitioning :\n"); -+ dev_err(kbdev->dev, "\tShader=%08x%08x\n", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG( -+ SHADER_PWRTRANS_HI), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG( -+ SHADER_PWRTRANS_LO), NULL)); -+ dev_err(kbdev->dev, "\tTiler =%08x%08x\n", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG( -+ TILER_PWRTRANS_HI), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG( -+ TILER_PWRTRANS_LO), NULL)); -+ dev_err(kbdev->dev, "\tL2 =%08x%08x\n", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG( -+ L2_PWRTRANS_HI), NULL), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG( -+ L2_PWRTRANS_LO), NULL)); -+#if KBASE_GPU_RESET_EN -+ dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); -+ if (kbase_prepare_to_reset_gpu(kbdev)) -+ kbase_reset_gpu(kbdev); -+#endif /* KBASE_GPU_RESET_EN */ -+ } else { -+ /* Log timelining information that a change in state has -+ * completed */ -+ kbase_timeline_pm_handle_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); -+ } -+} -+KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync); -+ -+void kbase_pm_enable_interrupts(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ /* -+ * Clear all interrupts, -+ * and unmask them all. -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, -+ NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, -+ NULL); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, -+ NULL); -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL); -+ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); -+ -+void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ /* -+ * Mask all interrupts, -+ * and clear them all. -+ */ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, -+ NULL); -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL); -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, -+ NULL); -+ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL); -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL); -+} -+ -+void kbase_pm_disable_interrupts(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_pm_disable_interrupts_nolock(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); -+ -+ -+/* -+ * pmu layout: -+ * 0x0000: PMU TAG (RO) (0xCAFECAFE) -+ * 0x0004: PMU VERSION ID (RO) (0x00000000) -+ * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE) -+ */ -+void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) -+{ -+ bool reset_required = is_resume; -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ if (kbdev->pm.backend.gpu_powered) { -+ /* Already turned on */ -+ if (kbdev->poweroff_pending) -+ kbase_pm_enable_interrupts(kbdev); -+ kbdev->poweroff_pending = false; -+ KBASE_DEBUG_ASSERT(!is_resume); -+ return; -+ } -+ -+ kbdev->poweroff_pending = false; -+ -+ KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u); -+ -+ if (is_resume && kbdev->pm.backend.callback_power_resume) { -+ kbdev->pm.backend.callback_power_resume(kbdev); -+ return; -+ } else if (kbdev->pm.backend.callback_power_on) { -+ kbdev->pm.backend.callback_power_on(kbdev); -+ /* If your platform properly keeps the GPU state you may use the -+ * return value of the callback_power_on function to -+ * conditionally reset the GPU on power up. Currently we are -+ * conservative and always reset the GPU. */ -+ reset_required = true; -+ } -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ kbdev->pm.backend.gpu_powered = true; -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (reset_required) { -+ /* GPU state was lost, reset GPU to ensure it is in a -+ * consistent state */ -+ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); -+ } -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_ctx_sched_restore_all_as(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ /* Lastly, enable the interrupts */ -+ kbase_pm_enable_interrupts(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_clock_on); -+ -+bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) -+{ -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ /* ASSERT that the cores should now be unavailable. No lock needed. */ -+ KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u); -+ -+ kbdev->poweroff_pending = true; -+ -+ if (!kbdev->pm.backend.gpu_powered) { -+ /* Already turned off */ -+ if (is_suspend && kbdev->pm.backend.callback_power_suspend) -+ kbdev->pm.backend.callback_power_suspend(kbdev); -+ return true; -+ } -+ -+ KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u); -+ -+ /* Disable interrupts. This also clears any outstanding interrupts */ -+ kbase_pm_disable_interrupts(kbdev); -+ /* Ensure that any IRQ handlers have finished */ -+ kbase_synchronize_irqs(kbdev); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (atomic_read(&kbdev->faults_pending)) { -+ /* Page/bus faults are still being processed. The GPU can not -+ * be powered off until they have completed */ -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ flags); -+ return false; -+ } -+ -+ kbase_pm_cache_snoop_disable(kbdev); -+ -+ /* The GPU power may be turned off from this point */ -+ kbdev->pm.backend.gpu_powered = false; -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags); -+ -+ if (is_suspend && kbdev->pm.backend.callback_power_suspend) -+ kbdev->pm.backend.callback_power_suspend(kbdev); -+ else if (kbdev->pm.backend.callback_power_off) -+ kbdev->pm.backend.callback_power_off(kbdev); -+ return true; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_clock_off); -+ -+struct kbasep_reset_timeout_data { -+ struct hrtimer timer; -+ bool timed_out; -+ struct kbase_device *kbdev; -+}; -+ -+void kbase_pm_reset_done(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ kbdev->pm.backend.reset_done = true; -+ wake_up(&kbdev->pm.backend.reset_done_wait); -+} -+ -+/** -+ * kbase_pm_wait_for_reset - Wait for a reset to happen -+ * -+ * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state. -+ * -+ * @kbdev: Kbase device -+ */ -+static void kbase_pm_wait_for_reset(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ wait_event(kbdev->pm.backend.reset_done_wait, -+ (kbdev->pm.backend.reset_done)); -+ kbdev->pm.backend.reset_done = false; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_reset_done); -+ -+static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) -+{ -+ struct kbasep_reset_timeout_data *rtdata = -+ container_of(timer, struct kbasep_reset_timeout_data, timer); -+ -+ rtdata->timed_out = 1; -+ -+ /* Set the wait queue to wake up kbase_pm_init_hw even though the reset -+ * hasn't completed */ -+ kbase_pm_reset_done(rtdata->kbdev); -+ -+ return HRTIMER_NORESTART; -+} -+ -+static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev) -+{ -+ struct device_node *np = kbdev->dev->of_node; -+ u32 jm_values[4]; -+ const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> -+ GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >> -+ GPU_ID_VERSION_MAJOR_SHIFT; -+ -+ kbdev->hw_quirks_sc = 0; -+ -+ /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. -+ * and needed due to MIDGLES-3539. See PRLAM-11035 */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) || -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035)) -+ kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE; -+ -+ /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. -+ */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) -+ kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD; -+ -+#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY -+ /* Enable alternative hardware counter selection if configured. */ -+ if (!GPU_ID_IS_NEW_FORMAT(prod_id)) -+ kbdev->hw_quirks_sc |= SC_ALT_COUNTERS; -+#endif -+ -+ /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797)) -+ kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS; -+ -+ if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) { -+ if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ -+ kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; -+ else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ -+ kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; -+ } -+ -+ if (!kbdev->hw_quirks_sc) -+ kbdev->hw_quirks_sc = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_CONFIG), NULL); -+ -+ kbdev->hw_quirks_tiler = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TILER_CONFIG), NULL); -+ -+ /* Set tiler clock gate override if required */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) -+ kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; -+ -+ /* Limit the GPU bus bandwidth if the platform needs this. */ -+ kbdev->hw_quirks_mmu = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_MMU_CONFIG), NULL); -+ -+ /* Limit read ID width for AXI */ -+ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS); -+ kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) << -+ L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT; -+ -+ /* Limit write ID width for AXI */ -+ kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES); -+ kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) << -+ L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT; -+ -+ if (kbdev->system_coherency == COHERENCY_ACE) { -+ /* Allow memory configuration disparity to be ignored, we -+ * optimize the use of shared memory and thus we expect -+ * some disparity in the memory configuration */ -+ kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY; -+ } -+ -+ kbdev->hw_quirks_jm = 0; -+ /* Only for T86x/T88x-based products after r2p0 */ -+ if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) { -+ -+ if (of_property_read_u32_array(np, -+ "jm_config", -+ &jm_values[0], -+ ARRAY_SIZE(jm_values))) { -+ /* Entry not in device tree, use defaults */ -+ jm_values[0] = 0; -+ jm_values[1] = 0; -+ jm_values[2] = 0; -+ jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; -+ } -+ -+ /* Limit throttle limit to 6 bits*/ -+ if (jm_values[3] > JM_MAX_JOB_THROTTLE_LIMIT) { -+ dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63)."); -+ jm_values[3] = JM_MAX_JOB_THROTTLE_LIMIT; -+ } -+ -+ /* Aggregate to one integer. */ -+ kbdev->hw_quirks_jm |= (jm_values[0] ? -+ JM_TIMESTAMP_OVERRIDE : 0); -+ kbdev->hw_quirks_jm |= (jm_values[1] ? -+ JM_CLOCK_GATE_OVERRIDE : 0); -+ kbdev->hw_quirks_jm |= (jm_values[2] ? -+ JM_JOB_THROTTLE_ENABLE : 0); -+ kbdev->hw_quirks_jm |= (jm_values[3] << -+ JM_JOB_THROTTLE_LIMIT_SHIFT); -+ -+ } else if (GPU_ID_IS_NEW_FORMAT(prod_id) && -+ (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == -+ GPU_ID2_PRODUCT_TMIX)) { -+ /* Only for tMIx */ -+ u32 coherency_features; -+ -+ coherency_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(COHERENCY_FEATURES), NULL); -+ -+ /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly -+ * documented for tMIx so force correct value here. -+ */ -+ if (coherency_features == -+ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { -+ kbdev->hw_quirks_jm |= -+ (COHERENCY_ACE_LITE | COHERENCY_ACE) << -+ JM_FORCE_COHERENCY_FEATURES_SHIFT; -+ } -+ } -+ -+ if (!kbdev->hw_quirks_jm) -+ kbdev->hw_quirks_jm = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(JM_CONFIG), NULL); -+ -+#ifdef CONFIG_MALI_CORESTACK -+#define MANUAL_POWER_CONTROL ((u32)(1 << 8)) -+ kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; -+#endif /* CONFIG_MALI_CORESTACK */ -+} -+ -+static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) -+{ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), -+ kbdev->hw_quirks_sc, NULL); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG), -+ kbdev->hw_quirks_tiler, NULL); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), -+ kbdev->hw_quirks_mmu, NULL); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), -+ kbdev->hw_quirks_jm, NULL); -+ -+} -+ -+void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) -+{ -+ if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && -+ !kbdev->cci_snoop_enabled) { -+#ifdef CONFIG_ARM64 -+ if (kbdev->snoop_enable_smc != 0) -+ kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); -+#endif /* CONFIG_ARM64 */ -+ dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); -+ kbdev->cci_snoop_enabled = true; -+ } -+} -+ -+void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) -+{ -+ if (kbdev->cci_snoop_enabled) { -+#ifdef CONFIG_ARM64 -+ if (kbdev->snoop_disable_smc != 0) { -+ mali_cci_flush_l2(kbdev); -+ kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); -+ } -+#endif /* CONFIG_ARM64 */ -+ dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); -+ kbdev->cci_snoop_enabled = false; -+ } -+} -+ -+static int kbase_pm_do_reset(struct kbase_device *kbdev) -+{ -+ struct kbasep_reset_timeout_data rtdata; -+ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); -+ -+ KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_SOFT_RESET, NULL); -+ -+ /* Unmask the reset complete interrupt only */ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, -+ NULL); -+ -+ /* Initialize a structure for tracking the status of the reset */ -+ rtdata.kbdev = kbdev; -+ rtdata.timed_out = 0; -+ -+ /* Create a timer to use as a timeout on the reset */ -+ hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ rtdata.timer.function = kbasep_reset_timeout; -+ -+ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), -+ HRTIMER_MODE_REL); -+ -+ /* Wait for the RESET_COMPLETED interrupt to be raised */ -+ kbase_pm_wait_for_reset(kbdev); -+ -+ if (rtdata.timed_out == 0) { -+ /* GPU has been reset */ -+ hrtimer_cancel(&rtdata.timer); -+ destroy_hrtimer_on_stack(&rtdata.timer); -+ return 0; -+ } -+ -+ /* No interrupt has been received - check if the RAWSTAT register says -+ * the reset has completed */ -+ if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & -+ RESET_COMPLETED) { -+ /* The interrupt is set in the RAWSTAT; this suggests that the -+ * interrupts are not getting to the CPU */ -+ dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); -+ /* If interrupts aren't working we can't continue. */ -+ destroy_hrtimer_on_stack(&rtdata.timer); -+ return -EINVAL; -+ } -+ -+ /* The GPU doesn't seem to be responding to the reset so try a hard -+ * reset */ -+ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", -+ RESET_TIMEOUT); -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_HARD_RESET, NULL); -+ -+ /* Restart the timer to wait for the hard reset to complete */ -+ rtdata.timed_out = 0; -+ -+ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), -+ HRTIMER_MODE_REL); -+ -+ /* Wait for the RESET_COMPLETED interrupt to be raised */ -+ kbase_pm_wait_for_reset(kbdev); -+ -+ if (rtdata.timed_out == 0) { -+ /* GPU has been reset */ -+ hrtimer_cancel(&rtdata.timer); -+ destroy_hrtimer_on_stack(&rtdata.timer); -+ return 0; -+ } -+ -+ destroy_hrtimer_on_stack(&rtdata.timer); -+ -+ dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", -+ RESET_TIMEOUT); -+ -+ return -EINVAL; -+} -+ -+static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) -+{ -+ struct kbase_device *kbdev = pdev->data; -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_SET_PROTECTED_MODE, NULL); -+ return 0; -+} -+ -+static int kbasep_protected_mode_disable(struct protected_mode_device *pdev) -+{ -+ struct kbase_device *kbdev = pdev->data; -+ -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ return kbase_pm_do_reset(kbdev); -+} -+ -+struct protected_mode_ops kbase_native_protected_ops = { -+ .protected_mode_enable = kbasep_protected_mode_enable, -+ .protected_mode_disable = kbasep_protected_mode_disable -+}; -+ -+int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) -+{ -+ unsigned long irq_flags; -+ int err; -+ bool resume_vinstr = false; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ /* Ensure the clock is on before attempting to access the hardware */ -+ if (!kbdev->pm.backend.gpu_powered) { -+ if (kbdev->pm.backend.callback_power_on) -+ kbdev->pm.backend.callback_power_on(kbdev); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, -+ irq_flags); -+ kbdev->pm.backend.gpu_powered = true; -+ spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, -+ irq_flags); -+ } -+ -+ /* Ensure interrupts are off to begin with, this also clears any -+ * outstanding interrupts */ -+ kbase_pm_disable_interrupts(kbdev); -+ /* Ensure cache snoops are disabled before reset. */ -+ kbase_pm_cache_snoop_disable(kbdev); -+ /* Prepare for the soft-reset */ -+ kbdev->pm.backend.reset_done = false; -+ -+ /* The cores should be made unavailable due to the reset */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ if (kbdev->shader_available_bitmap != 0u) -+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, -+ NULL, 0u, (u32)0u); -+ if (kbdev->tiler_available_bitmap != 0u) -+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, -+ NULL, NULL, 0u, (u32)0u); -+ kbdev->shader_available_bitmap = 0u; -+ kbdev->tiler_available_bitmap = 0u; -+ kbdev->l2_available_bitmap = 0u; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+ -+ /* Soft reset the GPU */ -+ if (kbdev->protected_mode_support) -+ err = kbdev->protected_ops->protected_mode_disable( -+ kbdev->protected_dev); -+ else -+ err = kbase_pm_do_reset(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ if (kbdev->protected_mode) -+ resume_vinstr = true; -+ kbdev->protected_mode = false; -+ kbase_ipa_model_use_configured_locked(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+ -+ if (err) -+ goto exit; -+ -+ if (flags & PM_HW_ISSUES_DETECT) -+ kbase_pm_hw_issues_detect(kbdev); -+ -+ kbase_pm_hw_issues_apply(kbdev); -+ kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); -+ -+ /* Sanity check protected mode was left after reset */ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { -+ u32 gpu_status = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_STATUS), NULL); -+ -+ WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE); -+ } -+ -+ /* If cycle counter was in use re-enable it, enable_irqs will only be -+ * false when called from kbase_pm_powerup */ -+ if (kbdev->pm.backend.gpu_cycle_counter_requests && -+ (flags & PM_ENABLE_IRQS)) { -+ /* enable interrupts as the L2 may have to be powered on */ -+ kbase_pm_enable_interrupts(kbdev); -+ kbase_pm_request_l2_caches(kbdev); -+ -+ /* Re-enable the counters if we need to */ -+ spin_lock_irqsave( -+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ irq_flags); -+ if (kbdev->pm.backend.gpu_cycle_counter_requests) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CYCLE_COUNT_START, NULL); -+ spin_unlock_irqrestore( -+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ irq_flags); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ kbase_pm_release_l2_caches(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+ -+ kbase_pm_disable_interrupts(kbdev); -+ } -+ -+ if (flags & PM_ENABLE_IRQS) -+ kbase_pm_enable_interrupts(kbdev); -+ -+exit: -+ /* If GPU is leaving protected mode resume vinstr operation. */ -+ if (kbdev->vinstr_ctx && resume_vinstr) -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+ -+ return err; -+} -+ -+/** -+ * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters -+ * -+ * Increase the count of cycle counter users and turn the cycle counters on if -+ * they were previously off -+ * -+ * This function is designed to be called by -+ * kbase_pm_request_gpu_cycle_counter() or -+ * kbase_pm_request_gpu_cycle_counter_l2_is_on() only -+ * -+ * When this function is called the l2 cache must be on and the l2 cache users -+ * count must have been incremented by a call to ( -+ * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() ) -+ * -+ * @kbdev: The kbase device structure of the device -+ */ -+static void -+kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ flags); -+ -+ ++kbdev->pm.backend.gpu_cycle_counter_requests; -+ -+ if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CYCLE_COUNT_START, NULL); -+ -+ spin_unlock_irqrestore( -+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ flags); -+} -+ -+void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < -+ INT_MAX); -+ -+ kbase_pm_request_l2_caches(kbdev); -+ -+ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter); -+ -+void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < -+ INT_MAX); -+ -+ kbase_pm_request_l2_caches_l2_is_on(kbdev); -+ -+ kbase_pm_request_gpu_cycle_counter_do_request(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on); -+ -+void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ flags); -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0); -+ -+ --kbdev->pm.backend.gpu_cycle_counter_requests; -+ -+ if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CYCLE_COUNT_STOP, NULL); -+ -+ spin_unlock_irqrestore( -+ &kbdev->pm.backend.gpu_cycle_counter_requests_lock, -+ flags); -+ -+ kbase_pm_release_l2_caches(kbdev); -+} -+ -+void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ kbase_pm_release_gpu_cycle_counter_nolock(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter); -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h -new file mode 100755 -index 000000000..6804f45ac ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h -@@ -0,0 +1,548 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Power management API definitions used internally by GPU backend -+ */ -+ -+#ifndef _KBASE_BACKEND_PM_INTERNAL_H_ -+#define _KBASE_BACKEND_PM_INTERNAL_H_ -+ -+#include -+ -+#include "mali_kbase_pm_ca.h" -+#include "mali_kbase_pm_policy.h" -+ -+ -+/** -+ * kbase_pm_dev_idle - The GPU is idle. -+ * -+ * The OS may choose to turn off idle devices -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_dev_idle(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_dev_activate - The GPU is active. -+ * -+ * The OS should avoid opportunistically turning off the GPU while it is active -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_dev_activate(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_get_present_cores - Get details of the cores that are present in -+ * the device. -+ * -+ * This function can be called by the active power policy to return a bitmask of -+ * the cores (of a specified type) present in the GPU device and also a count of -+ * the number of cores. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid -+ * pointer) -+ * @type: The type of core (see the enum kbase_pm_core_type enumeration) -+ * -+ * Return: The bit mask of cores present -+ */ -+u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type); -+ -+/** -+ * kbase_pm_get_active_cores - Get details of the cores that are currently -+ * active in the device. -+ * -+ * This function can be called by the active power policy to return a bitmask of -+ * the cores (of a specified type) that are actively processing work (i.e. -+ * turned on *and* busy). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @type: The type of core (see the enum kbase_pm_core_type enumeration) -+ * -+ * Return: The bit mask of active cores -+ */ -+u64 kbase_pm_get_active_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type); -+ -+/** -+ * kbase_pm_get_trans_cores - Get details of the cores that are currently -+ * transitioning between power states. -+ * -+ * This function can be called by the active power policy to return a bitmask of -+ * the cores (of a specified type) that are currently transitioning between -+ * power states. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @type: The type of core (see the enum kbase_pm_core_type enumeration) -+ * -+ * Return: The bit mask of transitioning cores -+ */ -+u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type); -+ -+/** -+ * kbase_pm_get_ready_cores - Get details of the cores that are currently -+ * powered and ready for jobs. -+ * -+ * This function can be called by the active power policy to return a bitmask of -+ * the cores (of a specified type) that are powered and ready for jobs (they may -+ * or may not be currently executing jobs). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @type: The type of core (see the enum kbase_pm_core_type enumeration) -+ * -+ * Return: The bit mask of ready cores -+ */ -+u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, -+ enum kbase_pm_core_type type); -+ -+/** -+ * kbase_pm_clock_on - Turn the clock for the device on, and enable device -+ * interrupts. -+ * -+ * This function can be used by a power policy to turn the clock for the GPU on. -+ * It should be modified during integration to perform the necessary actions to -+ * ensure that the GPU is fully powered and clocked. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid -+ * pointer) -+ * @is_resume: true if clock on due to resume after suspend, false otherwise -+ */ -+void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); -+ -+/** -+ * kbase_pm_clock_off - Disable device interrupts, and turn the clock for the -+ * device off. -+ * -+ * This function can be used by a power policy to turn the clock for the GPU -+ * off. It should be modified during integration to perform the necessary -+ * actions to turn the clock off (if this is possible in the integration). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid -+ * pointer) -+ * @is_suspend: true if clock off due to suspend, false otherwise -+ * -+ * Return: true if clock was turned off, or -+ * false if clock can not be turned off due to pending page/bus fault -+ * workers. Caller must flush MMU workqueues and retry -+ */ -+bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend); -+ -+/** -+ * kbase_pm_enable_interrupts - Enable interrupts on the device. -+ * -+ * Interrupts are also enabled after a call to kbase_pm_clock_on(). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_enable_interrupts(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_disable_interrupts - Disable interrupts on the device. -+ * -+ * This prevents delivery of Power Management interrupts to the CPU so that -+ * kbase_pm_check_transitions_nolock() will not be called from the IRQ handler -+ * until kbase_pm_enable_interrupts() or kbase_pm_clock_on() is called. -+ * -+ * Interrupts are also disabled after a call to kbase_pm_clock_off(). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_disable_interrupts(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_disable_interrupts_nolock - Version of kbase_pm_disable_interrupts() -+ * that does not take the hwaccess_lock -+ * -+ * Caller must hold the hwaccess_lock. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_init_hw - Initialize the hardware. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @flags: Flags specifying the type of PM init -+ * -+ * This function checks the GPU ID register to ensure that the GPU is supported -+ * by the driver and performs a reset on the device so that it is in a known -+ * state before the device is used. -+ * -+ * Return: 0 if the device is supported and successfully reset. -+ */ -+int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); -+ -+/** -+ * kbase_pm_reset_done - The GPU has been reset successfully. -+ * -+ * This function must be called by the GPU interrupt handler when the -+ * RESET_COMPLETED bit is set. It signals to the power management initialization -+ * code that the GPU has been successfully reset. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_reset_done(struct kbase_device *kbdev); -+ -+ -+/** -+ * kbase_pm_check_transitions_nolock - Check if there are any power transitions -+ * to make, and if so start them. -+ * -+ * This function will check the desired_xx_state members of -+ * struct kbase_pm_device_data and the actual status of the hardware to see if -+ * any power transitions can be made at this time to make the hardware state -+ * closer to the state desired by the power policy. -+ * -+ * The return value can be used to check whether all the desired cores are -+ * available, and so whether it's worth submitting a job (e.g. from a Power -+ * Management IRQ). -+ * -+ * Note that this still returns true when desired_xx_state has no -+ * cores. That is: of the no cores desired, none were *un*available. In -+ * this case, the caller may still need to try submitting jobs. This is because -+ * the Core Availability Policy might have taken us to an intermediate state -+ * where no cores are powered, before powering on more cores (e.g. for core -+ * rotation) -+ * -+ * The caller must hold kbase_device.pm.power_change_lock -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: non-zero when all desired cores are available. That is, -+ * it's worthwhile for the caller to submit a job. -+ * false otherwise -+ */ -+bool kbase_pm_check_transitions_nolock(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_check_transitions_sync - Synchronous and locking variant of -+ * kbase_pm_check_transitions_nolock() -+ * -+ * On returning, the desired state at the time of the call will have been met. -+ * -+ * There is nothing to stop the core being switched off by calls to -+ * kbase_pm_release_cores() or kbase_pm_unrequest_cores(). Therefore, the -+ * caller must have already made a call to -+ * kbase_pm_request_cores()/kbase_pm_request_cores_sync() previously. -+ * -+ * The usual use-case for this is to ensure cores are 'READY' after performing -+ * a GPU Reset. -+ * -+ * Unlike kbase_pm_check_transitions_nolock(), the caller must not hold -+ * kbase_device.pm.power_change_lock, because this function will take that -+ * lock itself. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_check_transitions_sync(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() -+ * where the caller must hold -+ * kbase_device.pm.power_change_lock -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_update_cores_state - Update the desired state of shader cores from -+ * the Power Policy, and begin any power -+ * transitions. -+ * -+ * This function will update the desired_xx_state members of -+ * struct kbase_pm_device_data by calling into the current Power Policy. It will -+ * then begin power transitions to make the hardware acheive the desired shader -+ * core state. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_update_cores_state(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_cancel_deferred_poweroff - Cancel any pending requests to power off -+ * the GPU and/or shader cores. -+ * -+ * This should be called by any functions which directly power off the GPU. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev); -+ -+/** -+ * kbasep_pm_init_core_use_bitmaps - Initialise data tracking the required -+ * and used cores. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbasep_pm_init_core_use_bitmaps(struct kbase_device *kbdev); -+ -+/** -+ * kbasep_pm_metrics_init - Initialize the metrics gathering framework. -+ * -+ * This must be called before other metric gathering APIs are called. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: 0 on success, error code on error -+ */ -+int kbasep_pm_metrics_init(struct kbase_device *kbdev); -+ -+/** -+ * kbasep_pm_metrics_term - Terminate the metrics gathering framework. -+ * -+ * This must be called when metric gathering is no longer required. It is an -+ * error to call any metrics gathering function (other than -+ * kbasep_pm_metrics_init()) after calling this function. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbasep_pm_metrics_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_report_vsync - Function to be called by the frame buffer driver to -+ * update the vsync metric. -+ * -+ * This function should be called by the frame buffer driver to update whether -+ * the system is hitting the vsync target or not. buffer_updated should be true -+ * if the vsync corresponded with a new frame being displayed, otherwise it -+ * should be false. This function does not need to be called every vsync, but -+ * only when the value of @buffer_updated differs from a previous call. -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ * @buffer_updated: True if the buffer has been updated on this VSync, -+ * false otherwise -+ */ -+void kbase_pm_report_vsync(struct kbase_device *kbdev, int buffer_updated); -+ -+/** -+ * kbase_pm_get_dvfs_action - Determine whether the DVFS system should change -+ * the clock speed of the GPU. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * This function should be called regularly by the DVFS system to check whether -+ * the clock speed of the GPU needs updating. -+ */ -+void kbase_pm_get_dvfs_action(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_request_gpu_cycle_counter - Mark that the GPU cycle counter is -+ * needed -+ * -+ * If the caller is the first caller then the GPU cycle counters will be enabled -+ * along with the l2 cache -+ * -+ * The GPU must be powered when calling this function (i.e. -+ * kbase_pm_context_active() must have been called). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_request_gpu_cycle_counter_l2_is_on - Mark GPU cycle counter is -+ * needed (l2 cache already on) -+ * -+ * This is a version of the above function -+ * (kbase_pm_request_gpu_cycle_counter()) suitable for being called when the -+ * l2 cache is known to be on and assured to be on until the subsequent call of -+ * kbase_pm_release_gpu_cycle_counter() such as when a job is submitted. It does -+ * not sleep and can be called from atomic functions. -+ * -+ * The GPU must be powered when calling this function (i.e. -+ * kbase_pm_context_active() must have been called) and the l2 cache must be -+ * powered on. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_release_gpu_cycle_counter - Mark that the GPU cycle counter is no -+ * longer in use -+ * -+ * If the caller is the last caller then the GPU cycle counters will be -+ * disabled. A request must have been made before a call to this. -+ * -+ * Caller must not hold the hwaccess_lock, as it will be taken in this function. -+ * If the caller is already holding this lock then -+ * kbase_pm_release_gpu_cycle_counter_nolock() must be used instead. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_release_gpu_cycle_counter_nolock - Version of kbase_pm_release_gpu_cycle_counter() -+ * that does not take hwaccess_lock -+ * -+ * Caller must hold the hwaccess_lock. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_wait_for_poweroff_complete - Wait for the poweroff workqueue to -+ * complete -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_register_access_enable - Enable access to GPU registers -+ * -+ * Enables access to the GPU registers before power management has powered up -+ * the GPU with kbase_pm_powerup(). -+ * -+ * Access to registers should be done using kbase_os_reg_read()/write() at this -+ * stage, not kbase_reg_read()/write(). -+ * -+ * This results in the power management callbacks provided in the driver -+ * configuration to get called to turn on power and/or clocks to the GPU. See -+ * kbase_pm_callback_conf. -+ * -+ * This should only be used before power management is powered up with -+ * kbase_pm_powerup() -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_register_access_enable(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_register_access_disable - Disable early register access -+ * -+ * Disables access to the GPU registers enabled earlier by a call to -+ * kbase_pm_register_access_enable(). -+ * -+ * This results in the power management callbacks provided in the driver -+ * configuration to get called to turn off power and/or clocks to the GPU. See -+ * kbase_pm_callback_conf -+ * -+ * This should only be used before power management is powered up with -+ * kbase_pm_powerup() -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_register_access_disable(struct kbase_device *kbdev); -+ -+/* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline -+ * function */ -+ -+/** -+ * kbase_pm_metrics_is_active - Check if the power management metrics -+ * collection is active. -+ * -+ * Note that this returns if the power management metrics collection was -+ * active at the time of calling, it is possible that after the call the metrics -+ * collection enable may have changed state. -+ * -+ * The caller must handle the consequence that the state may have changed. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * Return: true if metrics collection was active else false. -+ */ -+bool kbase_pm_metrics_is_active(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_do_poweron - Power on the GPU, and any cores that are requested. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid -+ * pointer) -+ * @is_resume: true if power on due to resume after suspend, -+ * false otherwise -+ */ -+void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); -+ -+/** -+ * kbase_pm_do_poweroff - Power off the GPU, and any cores that have been -+ * requested. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid -+ * pointer) -+ * @is_suspend: true if power off due to suspend, -+ * false otherwise -+ */ -+void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); -+ -+#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) -+void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, -+ unsigned long *total, unsigned long *busy); -+void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); -+#endif /* defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) */ -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ -+/** -+ * kbase_platform_dvfs_event - Report utilisation to DVFS code -+ * -+ * Function provided by platform specific code when DVFS is enabled to allow -+ * the power management metrics system to report utilisation. -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ * @utilisation: The current calculated utilisation by the metrics system. -+ * @util_gl_share: The current calculated gl share of utilisation. -+ * @util_cl_share: The current calculated cl share of utilisation per core -+ * group. -+ * Return: Returns 0 on failure and non zero on success. -+ */ -+ -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, -+ u32 util_gl_share, u32 util_cl_share[2]); -+#endif -+ -+void kbase_pm_power_changed(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_metrics_update - Inform the metrics system that an atom is either -+ * about to be run or has just completed. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @now: Pointer to the timestamp of the change, or NULL to use current time -+ * -+ * Caller must hold hwaccess_lock -+ */ -+void kbase_pm_metrics_update(struct kbase_device *kbdev, -+ ktime_t *now); -+ -+/** -+ * kbase_pm_cache_snoop_enable - Allow CPU snoops on the GPU -+ * If the GPU does not have coherency this is a no-op -+ * @kbdev: Device pointer -+ * -+ * This function should be called after L2 power up. -+ */ -+ -+void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_cache_snoop_disable - Prevent CPU snoops on the GPU -+ * If the GPU does not have coherency this is a no-op -+ * @kbdev: Device pointer -+ * -+ * This function should be called before L2 power off. -+ */ -+void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c -new file mode 100755 -index 000000000..024248ca7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c -@@ -0,0 +1,401 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Metrics for power management -+ */ -+ -+#include -+#include -+#include -+#include -+ -+/* When VSync is being hit aim for utilisation between 70-90% */ -+#define KBASE_PM_VSYNC_MIN_UTILISATION 70 -+#define KBASE_PM_VSYNC_MAX_UTILISATION 90 -+/* Otherwise aim for 10-40% */ -+#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 -+#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 -+ -+/* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns -+ * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly -+ * under 11s. Exceeding this will cause overflow */ -+#define KBASE_PM_TIME_SHIFT 8 -+ -+/* Maximum time between sampling of utilization data, without resetting the -+ * counters. */ -+#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */ -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) -+{ -+ unsigned long flags; -+ struct kbasep_pm_metrics_data *metrics; -+ -+ KBASE_DEBUG_ASSERT(timer != NULL); -+ -+ metrics = container_of(timer, struct kbasep_pm_metrics_data, timer); -+ kbase_pm_get_dvfs_action(metrics->kbdev); -+ -+ spin_lock_irqsave(&metrics->lock, flags); -+ -+ if (metrics->timer_active) -+ hrtimer_start(timer, -+ HR_TIMER_DELAY_MSEC(metrics->kbdev->pm.dvfs_period), -+ HRTIMER_MODE_REL); -+ -+ spin_unlock_irqrestore(&metrics->lock, flags); -+ -+ return HRTIMER_NORESTART; -+} -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -+ -+int kbasep_pm_metrics_init(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ kbdev->pm.backend.metrics.kbdev = kbdev; -+ -+ kbdev->pm.backend.metrics.time_period_start = ktime_get(); -+ kbdev->pm.backend.metrics.time_busy = 0; -+ kbdev->pm.backend.metrics.time_idle = 0; -+ kbdev->pm.backend.metrics.prev_busy = 0; -+ kbdev->pm.backend.metrics.prev_idle = 0; -+ kbdev->pm.backend.metrics.gpu_active = false; -+ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; -+ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; -+ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; -+ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; -+ kbdev->pm.backend.metrics.busy_cl[0] = 0; -+ kbdev->pm.backend.metrics.busy_cl[1] = 0; -+ kbdev->pm.backend.metrics.busy_gl = 0; -+ -+ spin_lock_init(&kbdev->pm.backend.metrics.lock); -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ kbdev->pm.backend.metrics.timer_active = true; -+ hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, -+ HRTIMER_MODE_REL); -+ kbdev->pm.backend.metrics.timer.function = dvfs_callback; -+ -+ hrtimer_start(&kbdev->pm.backend.metrics.timer, -+ HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), -+ HRTIMER_MODE_REL); -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); -+ -+void kbasep_pm_metrics_term(struct kbase_device *kbdev) -+{ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+ kbdev->pm.backend.metrics.timer_active = false; -+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -+ -+ hrtimer_cancel(&kbdev->pm.backend.metrics.timer); -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -+} -+ -+KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); -+ -+/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this -+ * function -+ */ -+static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, -+ ktime_t now) -+{ -+ ktime_t diff; -+ -+ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); -+ -+ diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); -+ if (ktime_to_ns(diff) < 0) -+ return; -+ -+ if (kbdev->pm.backend.metrics.gpu_active) { -+ u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); -+ -+ kbdev->pm.backend.metrics.time_busy += ns_time; -+ if (kbdev->pm.backend.metrics.active_cl_ctx[0]) -+ kbdev->pm.backend.metrics.busy_cl[0] += ns_time; -+ if (kbdev->pm.backend.metrics.active_cl_ctx[1]) -+ kbdev->pm.backend.metrics.busy_cl[1] += ns_time; -+ if (kbdev->pm.backend.metrics.active_gl_ctx[0]) -+ kbdev->pm.backend.metrics.busy_gl += ns_time; -+ if (kbdev->pm.backend.metrics.active_gl_ctx[1]) -+ kbdev->pm.backend.metrics.busy_gl += ns_time; -+ } else { -+ kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff) -+ >> KBASE_PM_TIME_SHIFT); -+ } -+ -+ kbdev->pm.backend.metrics.time_period_start = now; -+} -+ -+#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) -+/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this -+ * function. -+ */ -+static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, -+ ktime_t now) -+{ -+ /* Store previous value */ -+ kbdev->pm.backend.metrics.prev_idle = -+ kbdev->pm.backend.metrics.time_idle; -+ kbdev->pm.backend.metrics.prev_busy = -+ kbdev->pm.backend.metrics.time_busy; -+ -+ /* Reset current values */ -+ kbdev->pm.backend.metrics.time_period_start = now; -+ kbdev->pm.backend.metrics.time_idle = 0; -+ kbdev->pm.backend.metrics.time_busy = 0; -+ kbdev->pm.backend.metrics.busy_cl[0] = 0; -+ kbdev->pm.backend.metrics.busy_cl[1] = 0; -+ kbdev->pm.backend.metrics.busy_gl = 0; -+} -+ -+void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get()); -+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -+} -+ -+void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, -+ unsigned long *total_out, unsigned long *busy_out) -+{ -+ ktime_t now = ktime_get(); -+ unsigned long flags, busy, total; -+ -+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+ kbase_pm_get_dvfs_utilisation_calc(kbdev, now); -+ -+ busy = kbdev->pm.backend.metrics.time_busy; -+ total = busy + kbdev->pm.backend.metrics.time_idle; -+ -+ /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default -+ * 100ms) */ -+ if (total >= MALI_UTILIZATION_MAX_PERIOD) { -+ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); -+ } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { -+ total += kbdev->pm.backend.metrics.prev_idle + -+ kbdev->pm.backend.metrics.prev_busy; -+ busy += kbdev->pm.backend.metrics.prev_busy; -+ } -+ -+ *total_out = total; -+ *busy_out = busy; -+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -+} -+#endif -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ -+/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this -+ * function -+ */ -+int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, -+ int *util_gl_share, -+ int util_cl_share[2], -+ ktime_t now) -+{ -+ int utilisation; -+ int busy; -+ -+ kbase_pm_get_dvfs_utilisation_calc(kbdev, now); -+ -+ if (kbdev->pm.backend.metrics.time_idle + -+ kbdev->pm.backend.metrics.time_busy == 0) { -+ /* No data - so we return NOP */ -+ utilisation = -1; -+ if (util_gl_share) -+ *util_gl_share = -1; -+ if (util_cl_share) { -+ util_cl_share[0] = -1; -+ util_cl_share[1] = -1; -+ } -+ goto out; -+ } -+ -+ utilisation = (100 * kbdev->pm.backend.metrics.time_busy) / -+ (kbdev->pm.backend.metrics.time_idle + -+ kbdev->pm.backend.metrics.time_busy); -+ -+ busy = kbdev->pm.backend.metrics.busy_gl + -+ kbdev->pm.backend.metrics.busy_cl[0] + -+ kbdev->pm.backend.metrics.busy_cl[1]; -+ -+ if (busy != 0) { -+ if (util_gl_share) -+ *util_gl_share = -+ (100 * kbdev->pm.backend.metrics.busy_gl) / -+ busy; -+ if (util_cl_share) { -+ util_cl_share[0] = -+ (100 * kbdev->pm.backend.metrics.busy_cl[0]) / -+ busy; -+ util_cl_share[1] = -+ (100 * kbdev->pm.backend.metrics.busy_cl[1]) / -+ busy; -+ } -+ } else { -+ if (util_gl_share) -+ *util_gl_share = -1; -+ if (util_cl_share) { -+ util_cl_share[0] = -1; -+ util_cl_share[1] = -1; -+ } -+ } -+ -+out: -+ return utilisation; -+} -+ -+void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ int utilisation, util_gl_share; -+ int util_cl_share[2]; -+ ktime_t now; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+ -+ now = ktime_get(); -+ -+ utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, -+ util_cl_share, now); -+ -+ if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || -+ util_cl_share[1] < 0) { -+ utilisation = 0; -+ util_gl_share = 0; -+ util_cl_share[0] = 0; -+ util_cl_share[1] = 0; -+ goto out; -+ } -+ -+out: -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, -+ util_cl_share); -+#endif /*CONFIG_MALI_MIDGARD_DVFS */ -+ -+ kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); -+ -+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -+} -+ -+bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) -+{ -+ bool isactive; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+ isactive = kbdev->pm.backend.metrics.timer_active; -+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -+ -+ return isactive; -+} -+KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); -+ -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -+ -+/** -+ * kbase_pm_metrics_active_calc - Update PM active counts based on currently -+ * running atoms -+ * @kbdev: Device pointer -+ * -+ * The caller must hold kbdev->pm.backend.metrics.lock -+ */ -+static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) -+{ -+ int js; -+ -+ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); -+ -+ kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; -+ kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; -+ kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; -+ kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; -+ kbdev->pm.backend.metrics.gpu_active = false; -+ -+ for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { -+ struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); -+ -+ /* Head atom may have just completed, so if it isn't running -+ * then try the next atom */ -+ if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) -+ katom = kbase_gpu_inspect(kbdev, js, 1); -+ -+ if (katom && katom->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_SUBMITTED) { -+ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { -+ int device_nr = (katom->core_req & -+ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) -+ ? katom->device_nr : 0; -+ if (!WARN_ON(device_nr >= 2)) -+ kbdev->pm.backend.metrics. -+ active_cl_ctx[device_nr] = 1; -+ } else { -+ /* Slot 2 should not be running non-compute -+ * atoms */ -+ if (!WARN_ON(js >= 2)) -+ kbdev->pm.backend.metrics. -+ active_gl_ctx[js] = 1; -+ } -+ kbdev->pm.backend.metrics.gpu_active = true; -+ } -+ } -+} -+ -+/* called when job is submitted to or removed from a GPU slot */ -+void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) -+{ -+ unsigned long flags; -+ ktime_t now; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+ -+ if (!timestamp) { -+ now = ktime_get(); -+ timestamp = &now; -+ } -+ -+ /* Track how long CL and/or GL jobs have been busy for */ -+ kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); -+ -+ kbase_pm_metrics_active_calc(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -+} -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c -new file mode 100755 -index 000000000..075f020c6 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c -@@ -0,0 +1,973 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Power policy API implementations -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+static const struct kbase_pm_policy *const policy_list[] = { -+#ifdef CONFIG_MALI_NO_MALI -+ &kbase_pm_always_on_policy_ops, -+ &kbase_pm_demand_policy_ops, -+ &kbase_pm_coarse_demand_policy_ops, -+#if !MALI_CUSTOMER_RELEASE -+ &kbase_pm_demand_always_powered_policy_ops, -+ &kbase_pm_fast_start_policy_ops, -+#endif -+#else /* CONFIG_MALI_NO_MALI */ -+#if !PLATFORM_POWER_DOWN_ONLY -+ &kbase_pm_demand_policy_ops, -+#endif /* !PLATFORM_POWER_DOWN_ONLY */ -+ &kbase_pm_coarse_demand_policy_ops, -+ &kbase_pm_always_on_policy_ops, -+#if !MALI_CUSTOMER_RELEASE -+#if !PLATFORM_POWER_DOWN_ONLY -+ &kbase_pm_demand_always_powered_policy_ops, -+ &kbase_pm_fast_start_policy_ops, -+#endif /* !PLATFORM_POWER_DOWN_ONLY */ -+#endif -+#endif /* CONFIG_MALI_NO_MALI */ -+}; -+ -+/* The number of policies available in the system. -+ * This is derived from the number of functions listed in policy_get_functions. -+ */ -+#define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list)) -+ -+ -+/* Function IDs for looking up Timeline Trace codes in -+ * kbase_pm_change_state_trace_code */ -+enum kbase_pm_func_id { -+ KBASE_PM_FUNC_ID_REQUEST_CORES_START, -+ KBASE_PM_FUNC_ID_REQUEST_CORES_END, -+ KBASE_PM_FUNC_ID_RELEASE_CORES_START, -+ KBASE_PM_FUNC_ID_RELEASE_CORES_END, -+ /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither -+ * expect to hit it nor tend to hit it very much anyway. We can detect -+ * whether we need more instrumentation by a difference between -+ * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */ -+ -+ /* Must be the last */ -+ KBASE_PM_FUNC_ID_COUNT -+}; -+ -+ -+/* State changes during request/unrequest/release-ing cores */ -+enum { -+ KBASE_PM_CHANGE_STATE_SHADER = (1u << 0), -+ KBASE_PM_CHANGE_STATE_TILER = (1u << 1), -+ -+ /* These two must be last */ -+ KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER | -+ KBASE_PM_CHANGE_STATE_SHADER), -+ KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1 -+}; -+typedef u32 kbase_pm_change_state; -+ -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+/* Timeline Trace code lookups for each function */ -+static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT] -+ [KBASE_PM_CHANGE_STATE_COUNT] = { -+ /* kbase_pm_request_cores */ -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0, -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = -+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | -+ KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, -+ -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0, -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = -+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, -+ [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | -+ KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, -+ -+ /* kbase_pm_release_cores */ -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0, -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] = -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER | -+ KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, -+ -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0, -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] = -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, -+ [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER | -+ KBASE_PM_CHANGE_STATE_TILER] = -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END -+}; -+ -+static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, -+ enum kbase_pm_func_id func_id, -+ kbase_pm_change_state state) -+{ -+ int trace_code; -+ -+ KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT); -+ KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == -+ state); -+ -+ trace_code = kbase_pm_change_state_trace_code[func_id][state]; -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code); -+} -+ -+#else /* CONFIG_MALI_TRACE_TIMELINE */ -+static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev, -+ enum kbase_pm_func_id func_id, kbase_pm_change_state state) -+{ -+} -+ -+#endif /* CONFIG_MALI_TRACE_TIMELINE */ -+ -+/** -+ * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any -+ * requested shader cores -+ * @kbdev: Device pointer -+ */ -+static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev) -+{ -+ u64 prev_shader_state = kbdev->pm.backend.desired_shader_state; -+ u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->pm.backend.desired_shader_state &= -+ ~kbdev->pm.backend.shader_poweroff_pending; -+ kbdev->pm.backend.desired_tiler_state &= -+ ~kbdev->pm.backend.tiler_poweroff_pending; -+ -+ kbdev->pm.backend.shader_poweroff_pending = 0; -+ kbdev->pm.backend.tiler_poweroff_pending = 0; -+ -+ if (prev_shader_state != kbdev->pm.backend.desired_shader_state || -+ prev_tiler_state != -+ kbdev->pm.backend.desired_tiler_state || -+ kbdev->pm.backend.ca_in_transition) { -+ bool cores_are_available; -+ -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START); -+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); -+ KBASE_TIMELINE_PM_CHECKTRANS(kbdev, -+ SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END); -+ -+ /* Don't need 'cores_are_available', -+ * because we don't return anything */ -+ CSTD_UNUSED(cores_are_available); -+ } -+} -+ -+static enum hrtimer_restart -+kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer) -+{ -+ struct kbase_device *kbdev; -+ unsigned long flags; -+ -+ kbdev = container_of(timer, struct kbase_device, -+ pm.backend.gpu_poweroff_timer); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* It is safe for this call to do nothing if the work item is already -+ * queued. The worker function will read the must up-to-date state of -+ * kbdev->pm.backend.gpu_poweroff_pending under lock. -+ * -+ * If a state change occurs while the worker function is processing, -+ * this call will succeed as a work item can be requeued once it has -+ * started processing. -+ */ -+ if (kbdev->pm.backend.gpu_poweroff_pending) -+ queue_work(kbdev->pm.backend.gpu_poweroff_wq, -+ &kbdev->pm.backend.gpu_poweroff_work); -+ -+ if (kbdev->pm.backend.shader_poweroff_pending || -+ kbdev->pm.backend.tiler_poweroff_pending) { -+ kbdev->pm.backend.shader_poweroff_pending_time--; -+ -+ KBASE_DEBUG_ASSERT( -+ kbdev->pm.backend.shader_poweroff_pending_time -+ >= 0); -+ -+ if (!kbdev->pm.backend.shader_poweroff_pending_time) -+ kbasep_pm_do_poweroff_cores(kbdev); -+ } -+ -+ if (kbdev->pm.backend.poweroff_timer_needed) { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time); -+ -+ return HRTIMER_RESTART; -+ } -+ -+ kbdev->pm.backend.poweroff_timer_running = false; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return HRTIMER_NORESTART; -+} -+ -+static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data) -+{ -+ unsigned long flags; -+ struct kbase_device *kbdev; -+ bool do_poweroff = false; -+ -+ kbdev = container_of(data, struct kbase_device, -+ pm.backend.gpu_poweroff_work); -+ -+ mutex_lock(&kbdev->pm.lock); -+ -+ if (kbdev->pm.backend.gpu_poweroff_pending == 0) { -+ mutex_unlock(&kbdev->pm.lock); -+ return; -+ } -+ -+ kbdev->pm.backend.gpu_poweroff_pending--; -+ -+ if (kbdev->pm.backend.gpu_poweroff_pending > 0) { -+ mutex_unlock(&kbdev->pm.lock); -+ return; -+ } -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* Only power off the GPU if a request is still pending */ -+ if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev)) -+ do_poweroff = true; -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (do_poweroff) { -+ kbdev->pm.backend.poweroff_timer_needed = false; -+ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); -+ kbdev->pm.backend.poweroff_timer_running = false; -+ -+ /* Power off the GPU */ -+ kbase_pm_do_poweroff(kbdev, false); -+ } -+ -+ mutex_unlock(&kbdev->pm.lock); -+} -+ -+int kbase_pm_policy_init(struct kbase_device *kbdev) -+{ -+ struct workqueue_struct *wq; -+ -+ wq = alloc_workqueue("kbase_pm_do_poweroff", -+ WQ_HIGHPRI | WQ_UNBOUND, 1); -+ if (!wq) -+ return -ENOMEM; -+ -+ kbdev->pm.backend.gpu_poweroff_wq = wq; -+ INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work, -+ kbasep_pm_do_gpu_poweroff_wq); -+ hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer, -+ CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ kbdev->pm.backend.gpu_poweroff_timer.function = -+ kbasep_pm_do_gpu_poweroff_callback; -+ kbdev->pm.backend.pm_current_policy = policy_list[0]; -+ kbdev->pm.backend.pm_current_policy->init(kbdev); -+ kbdev->pm.gpu_poweroff_time = -+ HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); -+ kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; -+ kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU; -+ -+ return 0; -+} -+ -+void kbase_pm_policy_term(struct kbase_device *kbdev) -+{ -+ kbdev->pm.backend.pm_current_policy->term(kbdev); -+ destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq); -+} -+ -+void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ kbdev->pm.backend.poweroff_timer_needed = false; -+ hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.poweroff_timer_running = false; -+ -+ /* If wq is already running but is held off by pm.lock, make sure it has -+ * no effect */ -+ kbdev->pm.backend.gpu_poweroff_pending = 0; -+ -+ kbdev->pm.backend.shader_poweroff_pending = 0; -+ kbdev->pm.backend.tiler_poweroff_pending = 0; -+ kbdev->pm.backend.shader_poweroff_pending_time = 0; -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_pm_update_active(struct kbase_device *kbdev) -+{ -+ struct kbase_pm_device_data *pm = &kbdev->pm; -+ struct kbase_pm_backend_data *backend = &pm->backend; -+ unsigned long flags; -+ bool active; -+ -+ lockdep_assert_held(&pm->lock); -+ -+ /* pm_current_policy will never be NULL while pm.lock is held */ -+ KBASE_DEBUG_ASSERT(backend->pm_current_policy); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ active = backend->pm_current_policy->get_core_active(kbdev); -+ -+ if (active) { -+ if (backend->gpu_poweroff_pending) { -+ /* Cancel any pending power off request */ -+ backend->gpu_poweroff_pending = 0; -+ -+ /* If a request was pending then the GPU was still -+ * powered, so no need to continue */ -+ if (!kbdev->poweroff_pending) { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, -+ flags); -+ return; -+ } -+ } -+ -+ if (!backend->poweroff_timer_running && !backend->gpu_powered && -+ (pm->poweroff_gpu_ticks || -+ pm->poweroff_shader_ticks)) { -+ backend->poweroff_timer_needed = true; -+ backend->poweroff_timer_running = true; -+ hrtimer_start(&backend->gpu_poweroff_timer, -+ pm->gpu_poweroff_time, -+ HRTIMER_MODE_REL); -+ } -+ -+ /* Power on the GPU and any cores requested by the policy */ -+ if (pm->backend.poweroff_wait_in_progress) { -+ pm->backend.poweron_required = true; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } else { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ kbase_pm_do_poweron(kbdev, false); -+ } -+ } else { -+ /* It is an error for the power policy to power off the GPU -+ * when there are contexts active */ -+ KBASE_DEBUG_ASSERT(pm->active_count == 0); -+ -+ if (backend->shader_poweroff_pending || -+ backend->tiler_poweroff_pending) { -+ backend->shader_poweroff_pending = 0; -+ backend->tiler_poweroff_pending = 0; -+ backend->shader_poweroff_pending_time = 0; -+ } -+ -+ /* Request power off */ -+ if (pm->backend.gpu_powered) { -+ if (pm->poweroff_gpu_ticks) { -+ backend->gpu_poweroff_pending = -+ pm->poweroff_gpu_ticks; -+ backend->poweroff_timer_needed = true; -+ if (!backend->poweroff_timer_running) { -+ /* Start timer if not running (eg if -+ * power policy has been changed from -+ * always_on to something else). This -+ * will ensure the GPU is actually -+ * powered off */ -+ backend->poweroff_timer_running -+ = true; -+ hrtimer_start( -+ &backend->gpu_poweroff_timer, -+ pm->gpu_poweroff_time, -+ HRTIMER_MODE_REL); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, -+ flags); -+ } else { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, -+ flags); -+ -+ /* Power off the GPU immediately */ -+ kbase_pm_do_poweroff(kbdev, false); -+ } -+ } else { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+ } -+} -+ -+void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) -+{ -+ u64 desired_bitmap; -+ u64 desired_tiler_bitmap; -+ bool cores_are_available; -+ bool do_poweroff = false; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (kbdev->pm.backend.pm_current_policy == NULL) -+ return; -+ if (kbdev->pm.backend.poweroff_wait_in_progress) -+ return; -+ -+ if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap && -+ !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt -+ && !kbdev->tiler_inuse_cnt) { -+ /* We are trying to change in/out of protected mode - force all -+ * cores off so that the L2 powers down */ -+ desired_bitmap = 0; -+ desired_tiler_bitmap = 0; -+ } else { -+ desired_bitmap = -+ kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev); -+ desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev); -+ -+ if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0) -+ desired_tiler_bitmap = 1; -+ else -+ desired_tiler_bitmap = 0; -+ -+ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) { -+ /* Unless XAFFINITY is supported, enable core 0 if tiler -+ * required, regardless of core availability */ -+ if (kbdev->tiler_needed_cnt > 0 || -+ kbdev->tiler_inuse_cnt > 0) -+ desired_bitmap |= 1; -+ } -+ } -+ -+ if (kbdev->pm.backend.desired_shader_state != desired_bitmap) -+ KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, -+ (u32)desired_bitmap); -+ /* Are any cores being powered on? */ -+ if (~kbdev->pm.backend.desired_shader_state & desired_bitmap || -+ ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap || -+ kbdev->pm.backend.ca_in_transition) { -+ /* Check if we are powering off any cores before updating shader -+ * state */ -+ if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || -+ kbdev->pm.backend.desired_tiler_state & -+ ~desired_tiler_bitmap) { -+ /* Start timer to power off cores */ -+ kbdev->pm.backend.shader_poweroff_pending |= -+ (kbdev->pm.backend.desired_shader_state & -+ ~desired_bitmap); -+ kbdev->pm.backend.tiler_poweroff_pending |= -+ (kbdev->pm.backend.desired_tiler_state & -+ ~desired_tiler_bitmap); -+ -+ if (kbdev->pm.poweroff_shader_ticks && -+ !kbdev->protected_mode_transition) -+ kbdev->pm.backend.shader_poweroff_pending_time = -+ kbdev->pm.poweroff_shader_ticks; -+ else -+ do_poweroff = true; -+ } -+ -+ kbdev->pm.backend.desired_shader_state = desired_bitmap; -+ kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap; -+ -+ /* If any cores are being powered on, transition immediately */ -+ cores_are_available = kbase_pm_check_transitions_nolock(kbdev); -+ } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap || -+ kbdev->pm.backend.desired_tiler_state & -+ ~desired_tiler_bitmap) { -+ /* Start timer to power off cores */ -+ kbdev->pm.backend.shader_poweroff_pending |= -+ (kbdev->pm.backend.desired_shader_state & -+ ~desired_bitmap); -+ kbdev->pm.backend.tiler_poweroff_pending |= -+ (kbdev->pm.backend.desired_tiler_state & -+ ~desired_tiler_bitmap); -+ if (kbdev->pm.poweroff_shader_ticks && -+ !kbdev->protected_mode_transition) -+ kbdev->pm.backend.shader_poweroff_pending_time = -+ kbdev->pm.poweroff_shader_ticks; -+ else -+ kbasep_pm_do_poweroff_cores(kbdev); -+ } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && -+ desired_tiler_bitmap != 0 && -+ kbdev->pm.backend.poweroff_timer_needed) { -+ /* If power policy is keeping cores on despite there being no -+ * active contexts then disable poweroff timer as it isn't -+ * required. -+ * Only reset poweroff_timer_needed if we're not in the middle -+ * of the power off callback */ -+ kbdev->pm.backend.poweroff_timer_needed = false; -+ } -+ -+ /* Ensure timer does not power off wanted cores and make sure to power -+ * off unwanted cores */ -+ if (kbdev->pm.backend.shader_poweroff_pending || -+ kbdev->pm.backend.tiler_poweroff_pending) { -+ kbdev->pm.backend.shader_poweroff_pending &= -+ ~(kbdev->pm.backend.desired_shader_state & -+ desired_bitmap); -+ kbdev->pm.backend.tiler_poweroff_pending &= -+ ~(kbdev->pm.backend.desired_tiler_state & -+ desired_tiler_bitmap); -+ -+ if (!kbdev->pm.backend.shader_poweroff_pending && -+ !kbdev->pm.backend.tiler_poweroff_pending) -+ kbdev->pm.backend.shader_poweroff_pending_time = 0; -+ } -+ -+ /* Shader poweroff is deferred to the end of the function, to eliminate -+ * issues caused by the core availability policy recursing into this -+ * function */ -+ if (do_poweroff) -+ kbasep_pm_do_poweroff_cores(kbdev); -+ -+ /* Don't need 'cores_are_available', because we don't return anything */ -+ CSTD_UNUSED(cores_are_available); -+} -+ -+void kbase_pm_update_cores_state(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+int kbase_pm_list_policies(const struct kbase_pm_policy * const **list) -+{ -+ if (!list) -+ return POLICY_COUNT; -+ -+ *list = policy_list; -+ -+ return POLICY_COUNT; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_list_policies); -+ -+const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ return kbdev->pm.backend.pm_current_policy; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_get_policy); -+ -+void kbase_pm_set_policy(struct kbase_device *kbdev, -+ const struct kbase_pm_policy *new_policy) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ const struct kbase_pm_policy *old_policy; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(new_policy != NULL); -+ -+ KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id); -+ -+ /* During a policy change we pretend the GPU is active */ -+ /* A suspend won't happen here, because we're in a syscall from a -+ * userspace thread */ -+ kbase_pm_context_active(kbdev); -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+ /* Remove the policy to prevent IRQ handlers from working on it */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ old_policy = kbdev->pm.backend.pm_current_policy; -+ kbdev->pm.backend.pm_current_policy = NULL; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, -+ old_policy->id); -+ if (old_policy->term) -+ old_policy->term(kbdev); -+ -+ KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, -+ new_policy->id); -+ if (new_policy->init) -+ new_policy->init(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.pm_current_policy = new_policy; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* If any core power state changes were previously attempted, but -+ * couldn't be made because the policy was changing (current_policy was -+ * NULL), then re-try them here. */ -+ kbase_pm_update_active(kbdev); -+ kbase_pm_update_cores_state(kbdev); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ /* Now the policy change is finished, we release our fake context active -+ * reference */ -+ kbase_pm_context_idle(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_set_policy); -+ -+/* Check whether a state change has finished, and trace it as completed */ -+static void -+kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev) -+{ -+ if ((kbdev->shader_available_bitmap & -+ kbdev->pm.backend.desired_shader_state) -+ == kbdev->pm.backend.desired_shader_state && -+ (kbdev->tiler_available_bitmap & -+ kbdev->pm.backend.desired_tiler_state) -+ == kbdev->pm.backend.desired_tiler_state) -+ kbase_timeline_pm_check_handle_event(kbdev, -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED); -+} -+ -+void kbase_pm_request_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores) -+{ -+ u64 cores; -+ -+ kbase_pm_change_state change_gpu_state = 0u; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ cores = shader_cores; -+ while (cores) { -+ int bitnum = fls64(cores) - 1; -+ u64 bit = 1ULL << bitnum; -+ -+ /* It should be almost impossible for this to overflow. It would -+ * require 2^32 atoms to request a particular core, which would -+ * require 2^24 contexts to submit. This would require an amount -+ * of memory that is impossible on a 32-bit system and extremely -+ * unlikely on a 64-bit system. */ -+ int cnt = ++kbdev->shader_needed_cnt[bitnum]; -+ -+ if (1 == cnt) { -+ kbdev->shader_needed_bitmap |= bit; -+ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; -+ } -+ -+ cores &= ~bit; -+ } -+ -+ if (tiler_required) { -+ int cnt = ++kbdev->tiler_needed_cnt; -+ -+ if (1 == cnt) -+ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; -+ -+ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0); -+ } -+ -+ if (change_gpu_state) { -+ KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, -+ NULL, 0u, (u32) kbdev->shader_needed_bitmap); -+ -+ kbase_timeline_pm_cores_func(kbdev, -+ KBASE_PM_FUNC_ID_REQUEST_CORES_START, -+ change_gpu_state); -+ kbase_pm_update_cores_state_nolock(kbdev); -+ kbase_timeline_pm_cores_func(kbdev, -+ KBASE_PM_FUNC_ID_REQUEST_CORES_END, -+ change_gpu_state); -+ } -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_request_cores); -+ -+void kbase_pm_unrequest_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores) -+{ -+ kbase_pm_change_state change_gpu_state = 0u; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ while (shader_cores) { -+ int bitnum = fls64(shader_cores) - 1; -+ u64 bit = 1ULL << bitnum; -+ int cnt; -+ -+ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); -+ -+ cnt = --kbdev->shader_needed_cnt[bitnum]; -+ -+ if (0 == cnt) { -+ kbdev->shader_needed_bitmap &= ~bit; -+ -+ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; -+ } -+ -+ shader_cores &= ~bit; -+ } -+ -+ if (tiler_required) { -+ int cnt; -+ -+ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); -+ -+ cnt = --kbdev->tiler_needed_cnt; -+ -+ if (0 == cnt) -+ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; -+ } -+ -+ if (change_gpu_state) { -+ KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, -+ NULL, 0u, (u32) kbdev->shader_needed_bitmap); -+ -+ kbase_pm_update_cores_state_nolock(kbdev); -+ -+ /* Trace that any state change effectively completes immediately -+ * - no-one will wait on the state change */ -+ kbase_pm_trace_check_and_finish_state_change(kbdev); -+ } -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores); -+ -+enum kbase_pm_cores_ready -+kbase_pm_register_inuse_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores) -+{ -+ u64 prev_shader_needed; /* Just for tracing */ -+ u64 prev_shader_inuse; /* Just for tracing */ -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ prev_shader_needed = kbdev->shader_needed_bitmap; -+ prev_shader_inuse = kbdev->shader_inuse_bitmap; -+ -+ /* If desired_shader_state does not contain the requested cores, then -+ * power management is not attempting to powering those cores (most -+ * likely due to core availability policy) and a new job affinity must -+ * be chosen */ -+ if ((kbdev->pm.backend.desired_shader_state & shader_cores) != -+ shader_cores) { -+ return (kbdev->pm.backend.poweroff_wait_in_progress || -+ kbdev->pm.backend.pm_current_policy == NULL) ? -+ KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY; -+ } -+ -+ if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores || -+ (tiler_required && !kbdev->tiler_available_bitmap)) { -+ /* Trace ongoing core transition */ -+ kbase_timeline_pm_l2_transition_start(kbdev); -+ return KBASE_CORES_NOT_READY; -+ } -+ -+ /* If we started to trace a state change, then trace it has being -+ * finished by now, at the very latest */ -+ kbase_pm_trace_check_and_finish_state_change(kbdev); -+ /* Trace core transition done */ -+ kbase_timeline_pm_l2_transition_done(kbdev); -+ -+ while (shader_cores) { -+ int bitnum = fls64(shader_cores) - 1; -+ u64 bit = 1ULL << bitnum; -+ int cnt; -+ -+ KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0); -+ -+ cnt = --kbdev->shader_needed_cnt[bitnum]; -+ -+ if (0 == cnt) -+ kbdev->shader_needed_bitmap &= ~bit; -+ -+ /* shader_inuse_cnt should not overflow because there can only -+ * be a very limited number of jobs on the h/w at one time */ -+ -+ kbdev->shader_inuse_cnt[bitnum]++; -+ kbdev->shader_inuse_bitmap |= bit; -+ -+ shader_cores &= ~bit; -+ } -+ -+ if (tiler_required) { -+ KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0); -+ -+ --kbdev->tiler_needed_cnt; -+ -+ kbdev->tiler_inuse_cnt++; -+ -+ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0); -+ } -+ -+ if (prev_shader_needed != kbdev->shader_needed_bitmap) -+ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, -+ NULL, 0u, (u32) kbdev->shader_needed_bitmap); -+ -+ if (prev_shader_inuse != kbdev->shader_inuse_bitmap) -+ KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, -+ NULL, 0u, (u32) kbdev->shader_inuse_bitmap); -+ -+ return KBASE_CORES_READY; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores); -+ -+void kbase_pm_release_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores) -+{ -+ kbase_pm_change_state change_gpu_state = 0u; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ while (shader_cores) { -+ int bitnum = fls64(shader_cores) - 1; -+ u64 bit = 1ULL << bitnum; -+ int cnt; -+ -+ KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0); -+ -+ cnt = --kbdev->shader_inuse_cnt[bitnum]; -+ -+ if (0 == cnt) { -+ kbdev->shader_inuse_bitmap &= ~bit; -+ change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER; -+ } -+ -+ shader_cores &= ~bit; -+ } -+ -+ if (tiler_required) { -+ int cnt; -+ -+ KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0); -+ -+ cnt = --kbdev->tiler_inuse_cnt; -+ -+ if (0 == cnt) -+ change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER; -+ } -+ -+ if (change_gpu_state) { -+ KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, -+ NULL, 0u, (u32) kbdev->shader_inuse_bitmap); -+ -+ kbase_timeline_pm_cores_func(kbdev, -+ KBASE_PM_FUNC_ID_RELEASE_CORES_START, -+ change_gpu_state); -+ kbase_pm_update_cores_state_nolock(kbdev); -+ kbase_timeline_pm_cores_func(kbdev, -+ KBASE_PM_FUNC_ID_RELEASE_CORES_END, -+ change_gpu_state); -+ -+ /* Trace that any state change completed immediately */ -+ kbase_pm_trace_check_and_finish_state_change(kbdev); -+ } -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_release_cores); -+ -+void kbase_pm_request_cores_sync(struct kbase_device *kbdev, -+ bool tiler_required, -+ u64 shader_cores) -+{ -+ unsigned long flags; -+ -+ kbase_pm_wait_for_poweroff_complete(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_pm_request_cores(kbdev, tiler_required, shader_cores); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ kbase_pm_check_transitions_sync(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync); -+ -+void kbase_pm_request_l2_caches(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ u32 prior_l2_users_count; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ prior_l2_users_count = kbdev->l2_users_count++; -+ -+ KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0); -+ -+ /* if the GPU is reset while the l2 is on, l2 will be off but -+ * prior_l2_users_count will be > 0. l2_available_bitmap will have been -+ * set to 0 though by kbase_pm_init_hw */ -+ if (!prior_l2_users_count || !kbdev->l2_available_bitmap) -+ kbase_pm_check_transitions_nolock(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ wait_event(kbdev->pm.backend.l2_powered_wait, -+ kbdev->pm.backend.l2_powered == 1); -+ -+ /* Trace that any state change completed immediately */ -+ kbase_pm_trace_check_and_finish_state_change(kbdev); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches); -+ -+void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->l2_users_count++; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on); -+ -+void kbase_pm_release_l2_caches(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0); -+ -+ --kbdev->l2_users_count; -+ -+ if (!kbdev->l2_users_count) { -+ kbase_pm_check_transitions_nolock(kbdev); -+ /* Trace that any state change completed immediately */ -+ kbase_pm_trace_check_and_finish_state_change(kbdev); -+ } -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches); -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h -new file mode 100755 -index 000000000..611a90e66 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h -@@ -0,0 +1,227 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Power policy API definitions -+ */ -+ -+#ifndef _KBASE_PM_POLICY_H_ -+#define _KBASE_PM_POLICY_H_ -+ -+/** -+ * kbase_pm_policy_init - Initialize power policy framework -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Must be called before calling any other policy function -+ * -+ * Return: 0 if the power policy framework was successfully -+ * initialized, -errno otherwise. -+ */ -+int kbase_pm_policy_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_policy_term - Terminate power policy framework -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_policy_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_update_active - Update the active power state of the GPU -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Calls into the current power policy -+ */ -+void kbase_pm_update_active(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_update_cores - Update the desired core state of the GPU -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Calls into the current power policy -+ */ -+void kbase_pm_update_cores(struct kbase_device *kbdev); -+ -+ -+enum kbase_pm_cores_ready { -+ KBASE_CORES_NOT_READY = 0, -+ KBASE_NEW_AFFINITY = 1, -+ KBASE_CORES_READY = 2 -+}; -+ -+ -+/** -+ * kbase_pm_request_cores_sync - Synchronous variant of kbase_pm_request_cores() -+ * -+ * @kbdev: The kbase device structure for the device -+ * @tiler_required: true if the tiler is required, false otherwise -+ * @shader_cores: A bitmask of shader cores which are necessary for the job -+ * -+ * When this function returns, the @shader_cores will be in the READY state. -+ * -+ * This is safe variant of kbase_pm_check_transitions_sync(): it handles the -+ * work of ensuring the requested cores will remain powered until a matching -+ * call to kbase_pm_unrequest_cores()/kbase_pm_release_cores() (as appropriate) -+ * is made. -+ */ -+void kbase_pm_request_cores_sync(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores); -+ -+/** -+ * kbase_pm_request_cores - Mark one or more cores as being required -+ * for jobs to be submitted -+ * -+ * @kbdev: The kbase device structure for the device -+ * @tiler_required: true if the tiler is required, false otherwise -+ * @shader_cores: A bitmask of shader cores which are necessary for the job -+ * -+ * This function is called by the job scheduler to mark one or more cores as -+ * being required to submit jobs that are ready to run. -+ * -+ * The cores requested are reference counted and a subsequent call to -+ * kbase_pm_register_inuse_cores() or kbase_pm_unrequest_cores() should be -+ * made to dereference the cores as being 'needed'. -+ * -+ * The active power policy will meet or exceed the requirements of the -+ * requested cores in the system. Any core transitions needed will be begun -+ * immediately, but they might not complete/the cores might not be available -+ * until a Power Management IRQ. -+ * -+ * Return: 0 if the cores were successfully requested, or -errno otherwise. -+ */ -+void kbase_pm_request_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores); -+ -+/** -+ * kbase_pm_unrequest_cores - Unmark one or more cores as being required for -+ * jobs to be submitted. -+ * -+ * @kbdev: The kbase device structure for the device -+ * @tiler_required: true if the tiler is required, false otherwise -+ * @shader_cores: A bitmask of shader cores (as given to -+ * kbase_pm_request_cores() ) -+ * -+ * This function undoes the effect of kbase_pm_request_cores(). It should be -+ * used when a job is not going to be submitted to the hardware (e.g. the job is -+ * cancelled before it is enqueued). -+ * -+ * The active power policy will meet or exceed the requirements of the -+ * requested cores in the system. Any core transitions needed will be begun -+ * immediately, but they might not complete until a Power Management IRQ. -+ * -+ * The policy may use this as an indication that it can power down cores. -+ */ -+void kbase_pm_unrequest_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores); -+ -+/** -+ * kbase_pm_register_inuse_cores - Register a set of cores as in use by a job -+ * -+ * @kbdev: The kbase device structure for the device -+ * @tiler_required: true if the tiler is required, false otherwise -+ * @shader_cores: A bitmask of shader cores (as given to -+ * kbase_pm_request_cores() ) -+ * -+ * This function should be called after kbase_pm_request_cores() when the job -+ * is about to be submitted to the hardware. It will check that the necessary -+ * cores are available and if so update the 'needed' and 'inuse' bitmasks to -+ * reflect that the job is now committed to being run. -+ * -+ * If the necessary cores are not currently available then the function will -+ * return %KBASE_CORES_NOT_READY and have no effect. -+ * -+ * Return: %KBASE_CORES_NOT_READY if the cores are not immediately ready, -+ * -+ * %KBASE_NEW_AFFINITY if the affinity requested is not allowed, -+ * -+ * %KBASE_CORES_READY if the cores requested are already available -+ */ -+enum kbase_pm_cores_ready kbase_pm_register_inuse_cores( -+ struct kbase_device *kbdev, -+ bool tiler_required, -+ u64 shader_cores); -+ -+/** -+ * kbase_pm_release_cores - Release cores after a job has run -+ * -+ * @kbdev: The kbase device structure for the device -+ * @tiler_required: true if the tiler is required, false otherwise -+ * @shader_cores: A bitmask of shader cores (as given to -+ * kbase_pm_register_inuse_cores() ) -+ * -+ * This function should be called when a job has finished running on the -+ * hardware. A call to kbase_pm_register_inuse_cores() must have previously -+ * occurred. The reference counts of the specified cores will be decremented -+ * which may cause the bitmask of 'inuse' cores to be reduced. The power policy -+ * may then turn off any cores which are no longer 'inuse'. -+ */ -+void kbase_pm_release_cores(struct kbase_device *kbdev, -+ bool tiler_required, u64 shader_cores); -+ -+/** -+ * kbase_pm_request_l2_caches - Request l2 caches -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Request the use of l2 caches for all core groups, power up, wait and prevent -+ * the power manager from powering down the l2 caches. -+ * -+ * This tells the power management that the caches should be powered up, and -+ * they should remain powered, irrespective of the usage of shader cores. This -+ * does not return until the l2 caches are powered up. -+ * -+ * The caller must call kbase_pm_release_l2_caches() when they are finished -+ * to allow normal power management of the l2 caches to resume. -+ * -+ * This should only be used when power management is active. -+ */ -+void kbase_pm_request_l2_caches(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_request_l2_caches_l2_is_on - Request l2 caches but don't power on -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Increment the count of l2 users but do not attempt to power on the l2 -+ * -+ * It is the callers responsibility to ensure that the l2 is already powered up -+ * and to eventually call kbase_pm_release_l2_caches() -+ */ -+void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_request_l2_caches - Release l2 caches -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Release the use of l2 caches for all core groups and allow the power manager -+ * to power them down when necessary. -+ * -+ * This tells the power management that the caches can be powered down if -+ * necessary, with respect to the usage of shader cores. -+ * -+ * The caller must have called kbase_pm_request_l2_caches() prior to a call -+ * to this. -+ * -+ * This should only be used when power management is active. -+ */ -+void kbase_pm_release_l2_caches(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_PM_POLICY_H_ */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c -new file mode 100755 -index 000000000..d08c628dd ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c -@@ -0,0 +1,103 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+ -+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -+ u64 *system_time, struct timespec64 *ts) -+{ -+ u32 hi1, hi2; -+ -+ kbase_pm_request_gpu_cycle_counter(kbdev); -+ -+ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled -+ * correctly */ -+ do { -+ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), -+ NULL); -+ *cycle_counter = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); -+ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(CYCLE_COUNT_HI), -+ NULL); -+ *cycle_counter |= (((u64) hi1) << 32); -+ } while (hi1 != hi2); -+ -+ /* Read hi, lo, hi to ensure that overflow from lo to hi is handled -+ * correctly */ -+ do { -+ hi1 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), -+ NULL); -+ *system_time = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(TIMESTAMP_LO), NULL); -+ hi2 = kbase_reg_read(kbdev, GPU_CONTROL_REG(TIMESTAMP_HI), -+ NULL); -+ *system_time |= (((u64) hi1) << 32); -+ } while (hi1 != hi2); -+ -+ /* Record the CPU's idea of current time */ -+ ktime_get_raw_ts64(ts); -+ -+ kbase_pm_release_gpu_cycle_counter(kbdev); -+} -+ -+/** -+ * kbase_wait_write_flush - Wait for GPU write flush -+ * @kctx: Context pointer -+ * -+ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush -+ * its write buffer. -+ * -+ * Only in use for BASE_HW_ISSUE_6367 -+ * -+ * Note : If GPU resets occur then the counters are reset to zero, the delay may -+ * not be as expected. -+ */ -+#ifndef CONFIG_MALI_NO_MALI -+void kbase_wait_write_flush(struct kbase_context *kctx) -+{ -+ u32 base_count = 0; -+ -+ /* -+ * The caller must be holding onto the kctx or the call is from -+ * userspace. -+ */ -+ kbase_pm_context_active(kctx->kbdev); -+ kbase_pm_request_gpu_cycle_counter(kctx->kbdev); -+ -+ while (true) { -+ u32 new_count; -+ -+ new_count = kbase_reg_read(kctx->kbdev, -+ GPU_CONTROL_REG(CYCLE_COUNT_LO), NULL); -+ /* First time around, just store the count. */ -+ if (base_count == 0) { -+ base_count = new_count; -+ continue; -+ } -+ -+ /* No need to handle wrapping, unsigned maths works for this. */ -+ if ((new_count - base_count) > 1000) -+ break; -+ } -+ -+ kbase_pm_release_gpu_cycle_counter(kctx->kbdev); -+ kbase_pm_context_idle(kctx->kbdev); -+} -+#endif /* CONFIG_MALI_NO_MALI */ -diff --git a/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h -new file mode 100755 -index 000000000..433aa4b9c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.h -@@ -0,0 +1,52 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_BACKEND_TIME_H_ -+#define _KBASE_BACKEND_TIME_H_ -+ -+/** -+ * kbase_backend_get_gpu_time() - Get current GPU time -+ * @kbdev: Device pointer -+ * @cycle_counter: Pointer to u64 to store cycle counter in -+ * @system_time: Pointer to u64 to store system time in -+ * @ts: Pointer to struct timespec64 to store current monotonic -+ * time in -+ */ -+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -+ u64 *system_time, struct timespec64 *ts); -+ -+/** -+ * kbase_wait_write_flush() - Wait for GPU write flush -+ * @kctx: Context pointer -+ * -+ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush -+ * its write buffer. -+ * -+ * If GPU resets occur then the counters are reset to zero, the delay may not be -+ * as expected. -+ * -+ * This function is only in use for BASE_HW_ISSUE_6367 -+ */ -+#ifdef CONFIG_MALI_NO_MALI -+static inline void kbase_wait_write_flush(struct kbase_context *kctx) -+{ -+} -+#else -+void kbase_wait_write_flush(struct kbase_context *kctx); -+#endif -+ -+#endif /* _KBASE_BACKEND_TIME_H_ */ -diff --git a/drivers/gpu/arm/midgard/docs/Doxyfile b/drivers/gpu/arm/midgard/docs/Doxyfile -new file mode 100755 -index 000000000..35ff2f1ce ---- /dev/null -+++ b/drivers/gpu/arm/midgard/docs/Doxyfile -@@ -0,0 +1,126 @@ -+# -+# (C) COPYRIGHT 2011-2013, 2015 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ -+############################################################################## -+ -+# This file contains per-module Doxygen configuration. Please do not add -+# extra settings to this file without consulting all stakeholders, as they -+# may cause override project-wide settings. -+# -+# Additionally, when defining aliases, macros, sections etc, use the module -+# name as a prefix e.g. gles_my_alias. -+ -+############################################################################## -+ -+@INCLUDE = ../../bldsys/Doxyfile_common -+ -+# The INPUT tag can be used to specify the files and/or directories that contain -+# documented source files. You may enter file names like "myfile.cpp" or -+# directories like "/usr/src/myproject". Separate the files or directories -+# with spaces. -+ -+INPUT += ../../kernel/drivers/gpu/arm/midgard/ -+ -+############################################################################## -+# Everything below here is optional, and in most cases not required -+############################################################################## -+ -+# This tag can be used to specify a number of aliases that acts -+# as commands in the documentation. An alias has the form "name=value". -+# For example adding "sideeffect=\par Side Effects:\n" will allow you to -+# put the command \sideeffect (or @sideeffect) in the documentation, which -+# will result in a user-defined paragraph with heading "Side Effects:". -+# You can put \n's in the value part of an alias to insert newlines. -+ -+ALIASES += -+ -+# The ENABLED_SECTIONS tag can be used to enable conditional -+# documentation sections, marked by \if sectionname ... \endif. -+ -+ENABLED_SECTIONS += -+ -+# If the value of the INPUT tag contains directories, you can use the -+# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -+# and *.h) to filter out the source-files in the directories. If left -+# blank the following patterns are tested: -+# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx -+# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 -+ -+FILE_PATTERNS += -+ -+# The EXCLUDE tag can be used to specify files and/or directories that should -+# excluded from the INPUT source files. This way you can easily exclude a -+# subdirectory from a directory tree whose root is specified with the INPUT tag. -+EXCLUDE += ../../kernel/drivers/gpu/arm/midgard/platform ../../kernel/drivers/gpu/arm/midgard/platform_dummy ../../kernel/drivers/gpu/arm/midgard/scripts ../../kernel/drivers/gpu/arm/midgard/tests ../../kernel/drivers/gpu/arm/midgard/Makefile ../../kernel/drivers/gpu/arm/midgard/Makefile.kbase ../../kernel/drivers/gpu/arm/midgard/Kbuild ../../kernel/drivers/gpu/arm/midgard/Kconfig ../../kernel/drivers/gpu/arm/midgard/sconscript ../../kernel/drivers/gpu/arm/midgard/docs ../../kernel/drivers/gpu/arm/midgard/pm_test_script.sh ../../kernel/drivers/gpu/arm/midgard/mali_uk.h ../../kernel/drivers/gpu/arm/midgard/Makefile -+ -+ -+# If the value of the INPUT tag contains directories, you can use the -+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -+# certain files from those directories. Note that the wildcards are matched -+# against the file with absolute path, so to exclude all test directories -+# for example use the pattern */test/* -+ -+EXCLUDE_PATTERNS += -+ -+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -+# (namespaces, classes, functions, etc.) that should be excluded from the -+# output. The symbol name can be a fully qualified name, a word, or if the -+# wildcard * is used, a substring. Examples: ANamespace, AClass, -+# AClass::ANamespace, ANamespace::*Test -+ -+EXCLUDE_SYMBOLS += -+ -+# The EXAMPLE_PATH tag can be used to specify one or more files or -+# directories that contain example code fragments that are included (see -+# the \include command). -+ -+EXAMPLE_PATH += -+ -+# The IMAGE_PATH tag can be used to specify one or more files or -+# directories that contain image that are included in the documentation (see -+# the \image command). -+ -+IMAGE_PATH += -+ -+# The INCLUDE_PATH tag can be used to specify one or more directories that -+# contain include files that are not input files but should be processed by -+# the preprocessor. -+ -+INCLUDE_PATH += -+ -+# The PREDEFINED tag can be used to specify one or more macro names that -+# are defined before the preprocessor is started (similar to the -D option of -+# gcc). The argument of the tag is a list of macros of the form: name -+# or name=definition (no spaces). If the definition and the = are -+# omitted =1 is assumed. To prevent a macro definition from being -+# undefined via #undef or recursively expanded use the := operator -+# instead of the = operator. -+ -+PREDEFINED += -+ -+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -+# this tag can be used to specify a list of macro names that should be expanded. -+# The macro definition that is found in the sources will be used. -+# Use the PREDEFINED tag if you want to use a different macro definition. -+ -+EXPAND_AS_DEFINED += -+ -+# The DOTFILE_DIRS tag can be used to specify one or more directories that -+# contain dot files that are included in the documentation (see the -+# \dotfile command). -+ -+DOTFILE_DIRS += ../../kernel/drivers/gpu/arm/midgard/docs -+ -diff --git a/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot b/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot -new file mode 100755 -index 000000000..7ae05c2f8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/docs/policy_operation_diagram.dot -@@ -0,0 +1,112 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+digraph policy_objects_diagram { -+ rankdir=LR; -+ size="12,8"; -+ compound=true; -+ -+ node [ shape = box ]; -+ -+ subgraph cluster_policy_queues { -+ low_queue [ shape=record label = "LowP | {ctx_lo | ... | ctx_i | ... | ctx_hi}" ]; -+ queues_middle_sep [ label="" shape=plaintext width=0 height=0 ]; -+ -+ rt_queue [ shape=record label = "RT | {ctx_lo | ... | ctx_j | ... | ctx_hi}" ]; -+ -+ label = "Policy's Queue(s)"; -+ } -+ -+ call_enqueue [ shape=plaintext label="enqueue_ctx()" ]; -+ -+ { -+ rank=same; -+ ordering=out; -+ call_dequeue [ shape=plaintext label="dequeue_head_ctx()\n+ runpool_add_ctx()" ]; -+ call_ctxfinish [ shape=plaintext label="runpool_remove_ctx()" ]; -+ -+ call_ctxdone [ shape=plaintext label="don't requeue;\n/* ctx has no more jobs */" ]; -+ } -+ -+ subgraph cluster_runpool { -+ -+ as0 [ width=2 height = 0.25 label="AS0: Job_1, ..., Job_n" ]; -+ as1 [ width=2 height = 0.25 label="AS1: Job_1, ..., Job_m" ]; -+ as2 [ width=2 height = 0.25 label="AS2: Job_1, ..., Job_p" ]; -+ as3 [ width=2 height = 0.25 label="AS3: Job_1, ..., Job_q" ]; -+ -+ label = "Policy's Run Pool"; -+ } -+ -+ { -+ rank=same; -+ call_jdequeue [ shape=plaintext label="dequeue_job()" ]; -+ sstop_dotfixup [ shape=plaintext label="" width=0 height=0 ]; -+ } -+ -+ { -+ rank=same; -+ ordering=out; -+ sstop [ shape=ellipse label="SS-Timer expires" ] -+ jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; -+ -+ irq [ label="IRQ" shape=ellipse ]; -+ -+ job_finish [ shape=plaintext label="don't requeue;\n/* job done */" ]; -+ } -+ -+ hstop [ shape=ellipse label="HS-Timer expires" ] -+ -+ /* -+ * Edges -+ */ -+ -+ call_enqueue -> queues_middle_sep [ lhead=cluster_policy_queues ]; -+ -+ low_queue:qr -> call_dequeue:w; -+ rt_queue:qr -> call_dequeue:w; -+ -+ call_dequeue -> as1 [lhead=cluster_runpool]; -+ -+ as1->call_jdequeue [ltail=cluster_runpool]; -+ call_jdequeue->jobslots:0; -+ call_jdequeue->sstop_dotfixup [ arrowhead=none]; -+ sstop_dotfixup->sstop [label="Spawn SS-Timer"]; -+ sstop->jobslots [label="SoftStop"]; -+ sstop->hstop [label="Spawn HS-Timer"]; -+ hstop->jobslots:ne [label="HardStop"]; -+ -+ -+ as3->call_ctxfinish:ne [ ltail=cluster_runpool ]; -+ call_ctxfinish:sw->rt_queue:qm [ lhead=cluster_policy_queues label="enqueue_ctx()\n/* ctx still has jobs */" ]; -+ -+ call_ctxfinish->call_ctxdone [constraint=false]; -+ -+ call_ctxdone->call_enqueue [weight=0.1 labeldistance=20.0 labelangle=0.0 taillabel="Job submitted to the ctx" style=dotted constraint=false]; -+ -+ -+ { -+ jobslots->irq [constraint=false]; -+ -+ irq->job_finish [constraint=false]; -+ } -+ -+ irq->as2 [lhead=cluster_runpool label="requeue_job()\n/* timeslice expired */" ]; -+ -+} -diff --git a/drivers/gpu/arm/midgard/docs/policy_overview.dot b/drivers/gpu/arm/midgard/docs/policy_overview.dot -new file mode 100755 -index 000000000..159b993b7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/docs/policy_overview.dot -@@ -0,0 +1,63 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+digraph policy_objects_diagram { -+ rankdir=LR -+ size="6,6" -+ compound=true; -+ -+ node [ shape = box ]; -+ -+ call_enqueue [ shape=plaintext label="enqueue ctx" ]; -+ -+ -+ policy_queue [ label="Policy's Queue" ]; -+ -+ { -+ rank=same; -+ runpool [ label="Policy's Run Pool" ]; -+ -+ ctx_finish [ label="ctx finished" ]; -+ } -+ -+ { -+ rank=same; -+ jobslots [ shape=record label="Jobslots: | <0>js[0] | <1>js[1] | <2>js[2]" ]; -+ -+ job_finish [ label="Job finished" ]; -+ } -+ -+ -+ -+ /* -+ * Edges -+ */ -+ -+ call_enqueue -> policy_queue; -+ -+ policy_queue->runpool [label="dequeue ctx" weight=0.1]; -+ runpool->policy_queue [label="requeue ctx" weight=0.1]; -+ -+ runpool->ctx_finish [ style=dotted ]; -+ -+ runpool->jobslots [label="dequeue job" weight=0.1]; -+ jobslots->runpool [label="requeue job" weight=0.1]; -+ -+ jobslots->job_finish [ style=dotted ]; -+} -diff --git a/drivers/gpu/arm/midgard/ipa/Kbuild b/drivers/gpu/arm/midgard/ipa/Kbuild -new file mode 100755 -index 000000000..602b15f52 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/ipa/Kbuild -@@ -0,0 +1,24 @@ -+# -+# (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+midgard_kbase-y += \ -+ ipa/mali_kbase_ipa_simple.o \ -+ ipa/mali_kbase_ipa.o -+ -+midgard_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o -+ -+ifneq ($(wildcard $(src)/ipa/mali_kbase_ipa_tmix.c),) -+ midgard_kbase-y += ipa/mali_kbase_ipa_tmix.o -+endif -diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c -new file mode 100755 -index 000000000..01bdbb4e8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c -@@ -0,0 +1,585 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+#include -+#include -+#include -+#include "mali_kbase.h" -+#include "mali_kbase_ipa.h" -+#include "mali_kbase_ipa_debugfs.h" -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) -+#include -+#else -+#include -+#define dev_pm_opp_find_freq_exact opp_find_freq_exact -+#define dev_pm_opp_get_voltage opp_get_voltage -+#define dev_pm_opp opp -+#endif -+#include -+ -+#define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" -+ -+static struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { -+ &kbase_simple_ipa_model_ops, -+}; -+ -+int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) -+{ -+ int err = 0; -+ -+ lockdep_assert_held(&model->kbdev->ipa.lock); -+ -+ if (model->ops->recalculate) { -+ err = model->ops->recalculate(model); -+ if (err) { -+ dev_err(model->kbdev->dev, -+ "recalculation of power model %s returned error %d\n", -+ model->ops->name, err); -+ } -+ } -+ -+ return err; -+} -+ -+static struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, -+ const char *name) -+{ -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { -+ struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; -+ -+ if (!strcmp(ops->name, name)) -+ return ops; -+ } -+ -+ dev_err(kbdev->dev, "power model \'%s\' not found\n", name); -+ -+ return NULL; -+} -+ -+void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) -+{ -+ atomic_set(&kbdev->ipa_use_configured_model, false); -+} -+ -+void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) -+{ -+ atomic_set(&kbdev->ipa_use_configured_model, true); -+} -+ -+const char *kbase_ipa_model_name_from_id(u32 gpu_id) -+{ -+ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> -+ GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ -+ if (GPU_ID_IS_NEW_FORMAT(prod_id)) { -+ switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { -+ case GPU_ID2_PRODUCT_TMIX: -+ return KBASE_IPA_FALLBACK_MODEL_NAME; -+ default: -+ return KBASE_IPA_FALLBACK_MODEL_NAME; -+ } -+ } -+ -+ return KBASE_IPA_FALLBACK_MODEL_NAME; -+} -+ -+static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) -+{ -+ struct device_node *model_dt_node; -+ char compat_string[64]; -+ -+ snprintf(compat_string, sizeof(compat_string), "arm,%s", -+ model->ops->name); -+ -+ model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, -+ NULL, compat_string); -+ if (!model_dt_node && !model->missing_dt_node_warning) { -+ dev_warn(model->kbdev->dev, -+ "Couldn't find power_model DT node matching \'%s\'\n", -+ compat_string); -+ model->missing_dt_node_warning = true; -+ } -+ -+ return model_dt_node; -+} -+ -+int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, -+ const char *name, s32 *addr, -+ size_t num_elems, bool dt_required) -+{ -+ int err, i; -+ struct device_node *model_dt_node = get_model_dt_node(model); -+ char *origin; -+ -+ err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); -+ -+ if (err && dt_required) { -+ memset(addr, 0, sizeof(s32) * num_elems); -+ dev_warn(model->kbdev->dev, -+ "Error %d, no DT entry: %s.%s = %zu*[0]\n", -+ err, model->ops->name, name, num_elems); -+ origin = "zero"; -+ } else if (err && !dt_required) { -+ origin = "default"; -+ } else /* !err */ { -+ origin = "DT"; -+ } -+ -+ /* Create a unique debugfs entry for each element */ -+ for (i = 0; i < num_elems; ++i) { -+ char elem_name[32]; -+ -+ if (num_elems == 1) -+ snprintf(elem_name, sizeof(elem_name), "%s", name); -+ else -+ snprintf(elem_name, sizeof(elem_name), "%s.%d", -+ name, i); -+ -+ dev_dbg(model->kbdev->dev, "%s.%s = %d (%s)\n", -+ model->ops->name, elem_name, addr[i], origin); -+ -+ err = kbase_ipa_model_param_add(model, elem_name, -+ &addr[i], sizeof(s32), -+ PARAM_TYPE_S32); -+ if (err) -+ goto exit; -+ } -+exit: -+ return err; -+} -+ -+int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, -+ const char *name, char *addr, -+ size_t size, bool dt_required) -+{ -+ int err; -+ struct device_node *model_dt_node = get_model_dt_node(model); -+ const char *string_prop_value; -+ char *origin; -+ -+ err = of_property_read_string(model_dt_node, name, -+ &string_prop_value); -+ if (err && dt_required) { -+ strncpy(addr, "", size - 1); -+ dev_warn(model->kbdev->dev, -+ "Error %d, no DT entry: %s.%s = \'%s\'\n", -+ err, model->ops->name, name, addr); -+ err = 0; -+ origin = "zero"; -+ } else if (err && !dt_required) { -+ origin = "default"; -+ } else /* !err */ { -+ strncpy(addr, string_prop_value, size - 1); -+ origin = "DT"; -+ } -+ -+ addr[size - 1] = '\0'; -+ -+ dev_dbg(model->kbdev->dev, "%s.%s = \'%s\' (%s)\n", -+ model->ops->name, name, string_prop_value, origin); -+ -+ err = kbase_ipa_model_param_add(model, name, addr, size, -+ PARAM_TYPE_STRING); -+ -+ return err; -+} -+ -+void kbase_ipa_term_model(struct kbase_ipa_model *model) -+{ -+ if (!model) -+ return; -+ -+ lockdep_assert_held(&model->kbdev->ipa.lock); -+ -+ if (model->ops->term) -+ model->ops->term(model); -+ -+ kbase_ipa_model_param_free_all(model); -+ -+ kfree(model); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_term_model); -+ -+struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, -+ struct kbase_ipa_model_ops *ops) -+{ -+ struct kbase_ipa_model *model; -+ int err; -+ -+ lockdep_assert_held(&kbdev->ipa.lock); -+ -+ if (!ops || !ops->name) -+ return NULL; -+ -+ model = kzalloc(sizeof(struct kbase_ipa_model), GFP_KERNEL); -+ if (!model) -+ return NULL; -+ -+ model->kbdev = kbdev; -+ model->ops = ops; -+ INIT_LIST_HEAD(&model->params); -+ -+ err = model->ops->init(model); -+ if (err) { -+ dev_err(kbdev->dev, -+ "init of power model \'%s\' returned error %d\n", -+ ops->name, err); -+ goto term_model; -+ } -+ -+ err = kbase_ipa_model_recalculate(model); -+ if (err) -+ goto term_model; -+ -+ return model; -+ -+term_model: -+ kbase_ipa_term_model(model); -+ return NULL; -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_init_model); -+ -+static void kbase_ipa_term_locked(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->ipa.lock); -+ -+ /* Clean up the models */ -+ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) -+ kbase_ipa_term_model(kbdev->ipa.configured_model); -+ kbase_ipa_term_model(kbdev->ipa.fallback_model); -+ -+ kbdev->ipa.configured_model = NULL; -+ kbdev->ipa.fallback_model = NULL; -+} -+ -+int kbase_ipa_init(struct kbase_device *kbdev) -+{ -+ -+ const char *model_name; -+ struct kbase_ipa_model_ops *ops; -+ struct kbase_ipa_model *default_model = NULL; -+ int err; -+ -+ mutex_init(&kbdev->ipa.lock); -+ /* -+ * Lock during init to avoid warnings from lockdep_assert_held (there -+ * shouldn't be any concurrent access yet). -+ */ -+ mutex_lock(&kbdev->ipa.lock); -+ -+ /* The simple IPA model must *always* be present.*/ -+ ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); -+ -+ if (!ops->do_utilization_scaling_in_framework) { -+ dev_err(kbdev->dev, -+ "Fallback IPA model %s should not account for utilization\n", -+ ops->name); -+ err = -EINVAL; -+ goto end; -+ } -+ -+ default_model = kbase_ipa_init_model(kbdev, ops); -+ if (!default_model) { -+ err = -EINVAL; -+ goto end; -+ } -+ -+ kbdev->ipa.fallback_model = default_model; -+ err = of_property_read_string(kbdev->dev->of_node, -+ "ipa-model", -+ &model_name); -+ if (err) { -+ /* Attempt to load a match from GPU-ID */ -+ u32 gpu_id; -+ -+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ model_name = kbase_ipa_model_name_from_id(gpu_id); -+ dev_dbg(kbdev->dev, -+ "Inferring model from GPU ID 0x%x: \'%s\'\n", -+ gpu_id, model_name); -+ } else { -+ dev_dbg(kbdev->dev, -+ "Using ipa-model parameter from DT: \'%s\'\n", -+ model_name); -+ } -+ -+ if (strcmp(KBASE_IPA_FALLBACK_MODEL_NAME, model_name) != 0) { -+ ops = kbase_ipa_model_ops_find(kbdev, model_name); -+ kbdev->ipa.configured_model = kbase_ipa_init_model(kbdev, ops); -+ if (!kbdev->ipa.configured_model) { -+ err = -EINVAL; -+ goto end; -+ } -+ } else { -+ kbdev->ipa.configured_model = default_model; -+ err = 0; -+ } -+ -+ kbase_ipa_model_use_configured_locked(kbdev); -+ -+end: -+ if (err) -+ kbase_ipa_term_locked(kbdev); -+ else -+ dev_info(kbdev->dev, -+ "Using configured power model %s, and fallback %s\n", -+ kbdev->ipa.configured_model->ops->name, -+ kbdev->ipa.fallback_model->ops->name); -+ -+ mutex_unlock(&kbdev->ipa.lock); -+ return err; -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_init); -+ -+void kbase_ipa_term(struct kbase_device *kbdev) -+{ -+ mutex_lock(&kbdev->ipa.lock); -+ kbase_ipa_term_locked(kbdev); -+ mutex_unlock(&kbdev->ipa.lock); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_term); -+ -+/** -+ * kbase_scale_dynamic_power() - Scale a dynamic power coefficient to an OPP -+ * @c: Dynamic model coefficient, in pW/(Hz V^2). Should be in range -+ * 0 < c < 2^26 to prevent overflow. -+ * @freq: Frequency, in Hz. Range: 2^23 < freq < 2^30 (~8MHz to ~1GHz) -+ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) -+ * -+ * Keep a record of the approximate range of each value at every stage of the -+ * calculation, to ensure we don't overflow. This makes heavy use of the -+ * approximations 1000 = 2^10 and 1000000 = 2^20, but does the actual -+ * calculations in decimal for increased accuracy. -+ * -+ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) -+ */ -+static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, -+ const u32 voltage) -+{ -+ /* Range: 2^8 < v2 < 2^16 m(V^2) */ -+ const u32 v2 = (voltage * voltage) / 1000; -+ -+ /* Range: 2^3 < f_MHz < 2^10 MHz */ -+ const u32 f_MHz = freq / 1000000; -+ -+ /* Range: 2^11 < v2f_big < 2^26 kHz V^2 */ -+ const u32 v2f_big = v2 * f_MHz; -+ -+ /* Range: 2^1 < v2f < 2^16 MHz V^2 */ -+ const u32 v2f = v2f_big / 1000; -+ -+ /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. -+ * Must be < 2^42 to avoid overflowing the return value. */ -+ const u64 v2fc = (u64) c * (u64) v2f; -+ u32 remainder; -+ -+ /* Range: 0 < v2fc / 1000 < 2^13 mW */ -+ // static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) -+ return div_u64_rem(v2fc, 1000, &remainder); -+} -+ -+/** -+ * kbase_scale_static_power() - Scale a static power coefficient to an OPP -+ * @c: Static model coefficient, in uW/V^3. Should be in range -+ * 0 < c < 2^32 to prevent overflow. -+ * @voltage: Voltage, in mV. Range: 2^9 < voltage < 2^13 (~0.5V to ~8V) -+ * -+ * Return: Power consumption, in mW. Range: 0 < p < 2^13 (0W to ~8W) -+ */ -+u32 kbase_scale_static_power(const u32 c, const u32 voltage) -+{ -+ /* Range: 2^8 < v2 < 2^16 m(V^2) */ -+ const u32 v2 = (voltage * voltage) / 1000; -+ -+ /* Range: 2^17 < v3_big < 2^29 m(V^2) mV */ -+ const u32 v3_big = v2 * voltage; -+ -+ /* Range: 2^7 < v3 < 2^19 m(V^3) */ -+ const u32 v3 = v3_big / 1000; -+ -+ /* -+ * Range (working backwards from next line): 0 < v3c_big < 2^33 nW. -+ * The result should be < 2^52 to avoid overflowing the return value. -+ */ -+ const u64 v3c_big = (u64) c * (u64) v3; -+ u32 remainder; -+ -+ /* Range: 0 < v3c_big / 1000000 < 2^13 mW */ -+ // return v3c_big / 1000000; -+ return div_u64_rem(v3c_big, 1000000, &remainder); -+} -+ -+static struct kbase_ipa_model *get_current_model(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->ipa.lock); -+ -+ if (atomic_read(&kbdev->ipa_use_configured_model)) -+ return kbdev->ipa.configured_model; -+ else -+ return kbdev->ipa.fallback_model; -+} -+ -+static u32 get_static_power_locked(struct kbase_device *kbdev, -+ struct kbase_ipa_model *model, -+ unsigned long voltage) -+{ -+ u32 power = 0; -+ int err; -+ u32 power_coeff; -+ -+ lockdep_assert_held(&model->kbdev->ipa.lock); -+ -+ if (!model->ops->get_static_coeff) -+ model = kbdev->ipa.fallback_model; -+ -+ if (model->ops->get_static_coeff) { -+ err = model->ops->get_static_coeff(model, &power_coeff); -+ if (!err) -+ power = kbase_scale_static_power(power_coeff, -+ (u32) voltage); -+ } -+ -+ return power; -+} -+ -+#ifdef CONFIG_MALI_PWRSOFT_765 -+static unsigned long kbase_get_static_power(struct devfreq *df, -+ unsigned long voltage) -+#else -+static unsigned long kbase_get_static_power(unsigned long voltage) -+#endif -+{ -+ struct kbase_ipa_model *model; -+ u32 power = 0; -+#ifdef CONFIG_MALI_PWRSOFT_765 -+ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); -+#else -+ struct kbase_device *kbdev = kbase_find_device(-1); -+#endif -+ -+ mutex_lock(&kbdev->ipa.lock); -+ -+ model = get_current_model(kbdev); -+ power = get_static_power_locked(kbdev, model, voltage); -+ -+ mutex_unlock(&kbdev->ipa.lock); -+ -+#ifndef CONFIG_MALI_PWRSOFT_765 -+ kbase_release_device(kbdev); -+#endif -+ -+ return power; -+} -+ -+#ifdef CONFIG_MALI_PWRSOFT_765 -+static unsigned long kbase_get_dynamic_power(struct devfreq *df, -+ unsigned long freq, -+ unsigned long voltage) -+#else -+static unsigned long kbase_get_dynamic_power(unsigned long freq, -+ unsigned long voltage) -+#endif -+{ -+ struct kbase_ipa_model *model; -+ u32 power_coeff = 0, power = 0; -+ int err = 0; -+#ifdef CONFIG_MALI_PWRSOFT_765 -+ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); -+#else -+ struct kbase_device *kbdev = kbase_find_device(-1); -+#endif -+ -+ mutex_lock(&kbdev->ipa.lock); -+ -+ model = kbdev->ipa.fallback_model; -+ -+ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); -+ -+ if (!err) -+ power = kbase_scale_dynamic_power(power_coeff, freq, voltage); -+ else -+ dev_err_ratelimited(kbdev->dev, -+ "Model %s returned error code %d\n", -+ model->ops->name, err); -+ -+ mutex_unlock(&kbdev->ipa.lock); -+ -+#ifndef CONFIG_MALI_PWRSOFT_765 -+ kbase_release_device(kbdev); -+#endif -+ -+ return power; -+} -+ -+int kbase_get_real_power(struct devfreq *df, u32 *power, -+ unsigned long freq, -+ unsigned long voltage) -+{ -+ struct kbase_ipa_model *model; -+ u32 power_coeff = 0; -+ int err = 0; -+ struct kbase_device *kbdev = dev_get_drvdata(&df->dev); -+ -+ mutex_lock(&kbdev->ipa.lock); -+ -+ model = get_current_model(kbdev); -+ -+ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); -+ -+ /* If we switch to protected model between get_current_model() and -+ * get_dynamic_coeff(), counter reading could fail. If that happens -+ * (unlikely, but possible), revert to the fallback model. */ -+ if (err && model != kbdev->ipa.fallback_model) { -+ model = kbdev->ipa.fallback_model; -+ err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); -+ } -+ -+ if (err) -+ goto exit_unlock; -+ -+ *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); -+ -+ if (model->ops->do_utilization_scaling_in_framework) { -+ struct devfreq_dev_status *status = &df->last_status; -+ unsigned long total_time = max(status->total_time, 1ul); -+ u64 busy_time = min(status->busy_time, total_time); -+ u32 remainder; -+ -+ // *power = ((u64) *power * (u64) busy_time) / total_time; -+ *power = div_u64_rem(((u64) *power * (u64) busy_time), total_time, &remainder); -+ } -+ -+ *power += get_static_power_locked(kbdev, model, voltage); -+ -+exit_unlock: -+ mutex_unlock(&kbdev->ipa.lock); -+ -+ return err; -+} -+KBASE_EXPORT_TEST_API(kbase_get_real_power); -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -+struct devfreq_cooling_ops kbase_ipa_power_model_ops = { -+#else -+struct devfreq_cooling_power kbase_ipa_power_model_ops = { -+#endif -+ .get_static_power = &kbase_get_static_power, -+ .get_dynamic_power = &kbase_get_dynamic_power, -+}; -+KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); -diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h -new file mode 100755 -index 000000000..b2d3db149 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h -@@ -0,0 +1,148 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_IPA_H_ -+#define _KBASE_IPA_H_ -+ -+#if defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL) -+ -+struct devfreq; -+ -+struct kbase_ipa_model { -+ struct list_head link; -+ struct kbase_device *kbdev; -+ void *model_data; -+ struct kbase_ipa_model_ops *ops; -+ struct list_head params; -+ bool missing_dt_node_warning; -+}; -+ -+/** -+ * kbase_ipa_model_add_param_s32 - Add an integer model parameter -+ * @model: pointer to IPA model -+ * @name: name of corresponding debugfs entry -+ * @addr: address where the value is stored -+ * @num_elems: number of elements (1 if not an array) -+ * @dt_required: if false, a corresponding devicetree entry is not required, -+ * and the current value will be used. If true, a warning is -+ * output and the data is zeroed -+ * -+ * Return: 0 on success, or an error code -+ */ -+int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, -+ const char *name, s32 *addr, -+ size_t num_elems, bool dt_required); -+ -+/** -+ * kbase_ipa_model_add_param_string - Add a string model parameter -+ * @model: pointer to IPA model -+ * @name: name of corresponding debugfs entry -+ * @addr: address where the value is stored -+ * @size: size, in bytes, of the value storage (so the maximum string -+ * length is size - 1) -+ * @dt_required: if false, a corresponding devicetree entry is not required, -+ * and the current value will be used. If true, a warning is -+ * output and the data is zeroed -+ * -+ * Return: 0 on success, or an error code -+ */ -+int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, -+ const char *name, char *addr, -+ size_t size, bool dt_required); -+ -+struct kbase_ipa_model_ops { -+ char *name; -+ /* The init, recalculate and term ops on the default model are always -+ * called. However, all the other models are only invoked if the model -+ * is selected in the device tree. Otherwise they are never -+ * initialized. Additional resources can be acquired by models in -+ * init(), however they must be terminated in the term(). -+ */ -+ int (*init)(struct kbase_ipa_model *model); -+ /* Called immediately after init(), or when a parameter is changed, so -+ * that any coefficients derived from model parameters can be -+ * recalculated. */ -+ int (*recalculate)(struct kbase_ipa_model *model); -+ void (*term)(struct kbase_ipa_model *model); -+ /* -+ * get_dynamic_coeff() - calculate dynamic power coefficient -+ * @model: pointer to model -+ * @coeffp: pointer to return value location -+ * @current_freq: frequency the GPU has been running at for the -+ * previous sampling period. -+ * -+ * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which -+ * is then scaled by the IPA framework according to the current OPP's -+ * frequency and voltage. -+ * -+ * Return: 0 on success, or an error code. -+ */ -+ int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, -+ u32 current_freq); -+ /* -+ * get_static_coeff() - calculate static power coefficient -+ * @model: pointer to model -+ * @coeffp: pointer to return value location -+ * -+ * Calculate a static power coefficient, with units uW/(V^3), which is -+ * scaled by the IPA framework according to the current OPP's voltage. -+ * -+ * Return: 0 on success, or an error code. -+ */ -+ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); -+ /* If false, the model's get_dynamic_coeff() method accounts for how -+ * long the GPU was active over the sample period. If true, the -+ * framework will scale the calculated power according to the -+ * utilization stats recorded by devfreq in get_real_power(). */ -+ bool do_utilization_scaling_in_framework; -+}; -+ -+/* Models can be registered only in the platform's platform_init_func call */ -+int kbase_ipa_model_ops_register(struct kbase_device *kbdev, -+ struct kbase_ipa_model_ops *new_model_ops); -+struct kbase_ipa_model *kbase_ipa_get_model(struct kbase_device *kbdev, -+ const char *name); -+ -+int kbase_ipa_init(struct kbase_device *kbdev); -+void kbase_ipa_term(struct kbase_device *kbdev); -+void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev); -+void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev); -+int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); -+struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, -+ struct kbase_ipa_model_ops *ops); -+void kbase_ipa_term_model(struct kbase_ipa_model *model); -+ -+extern struct kbase_ipa_model_ops kbase_simple_ipa_model_ops; -+ -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -+extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; -+#else -+extern struct devfreq_cooling_power kbase_ipa_power_model_ops; -+#endif -+ -+#else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ -+ -+static inline void kbase_ipa_model_use_fallback_locked(struct kbase_device *kbdev) -+{ } -+ -+static inline void kbase_ipa_model_use_configured_locked(struct kbase_device *kbdev) -+{ } -+ -+#endif /* (defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c -new file mode 100755 -index 000000000..eafc14009 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c -@@ -0,0 +1,219 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+ -+#include "mali_kbase.h" -+#include "mali_kbase_ipa.h" -+#include "mali_kbase_ipa_debugfs.h" -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) -+#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE -+#endif -+ -+struct kbase_ipa_model_param { -+ char *name; -+ union { -+ void *voidp; -+ s32 *s32p; -+ char *str; -+ } addr; -+ size_t size; -+ enum kbase_ipa_model_param_type type; -+ struct kbase_ipa_model *model; -+ struct list_head link; -+}; -+ -+static int param_int_get(void *data, u64 *val) -+{ -+ struct kbase_ipa_model_param *param = data; -+ -+ mutex_lock(¶m->model->kbdev->ipa.lock); -+ *(s64 *) val = *param->addr.s32p; -+ mutex_unlock(¶m->model->kbdev->ipa.lock); -+ -+ return 0; -+} -+ -+static int param_int_set(void *data, u64 val) -+{ -+ struct kbase_ipa_model_param *param = data; -+ struct kbase_ipa_model *model = param->model; -+ s64 sval = (s64) val; -+ int err = 0; -+ -+ if (sval < S32_MIN || sval > S32_MAX) -+ return -ERANGE; -+ -+ mutex_lock(¶m->model->kbdev->ipa.lock); -+ *param->addr.s32p = val; -+ err = kbase_ipa_model_recalculate(model); -+ mutex_unlock(¶m->model->kbdev->ipa.lock); -+ -+ return err; -+} -+ -+DEFINE_DEBUGFS_ATTRIBUTE(fops_s32, param_int_get, param_int_set, "%lld\n"); -+ -+static ssize_t param_string_get(struct file *file, char __user *user_buf, -+ size_t count, loff_t *ppos) -+{ -+ struct kbase_ipa_model_param *param = file->private_data; -+ ssize_t ret; -+ size_t len; -+ -+ mutex_lock(¶m->model->kbdev->ipa.lock); -+ len = strnlen(param->addr.str, param->size - 1) + 1; -+ ret = simple_read_from_buffer(user_buf, count, ppos, -+ param->addr.str, len); -+ mutex_unlock(¶m->model->kbdev->ipa.lock); -+ -+ return ret; -+} -+ -+static ssize_t param_string_set(struct file *file, const char __user *user_buf, -+ size_t count, loff_t *ppos) -+{ -+ struct kbase_ipa_model_param *param = file->private_data; -+ struct kbase_ipa_model *model = param->model; -+ ssize_t ret = count; -+ size_t buf_size; -+ int err; -+ -+ mutex_lock(&model->kbdev->ipa.lock); -+ -+ if (count > param->size) { -+ ret = -EINVAL; -+ goto end; -+ } -+ -+ buf_size = min(param->size - 1, count); -+ if (copy_from_user(param->addr.str, user_buf, buf_size)) { -+ ret = -EFAULT; -+ goto end; -+ } -+ -+ param->addr.str[buf_size] = '\0'; -+ -+ err = kbase_ipa_model_recalculate(model); -+ if (err < 0) -+ ret = err; -+ -+end: -+ mutex_unlock(&model->kbdev->ipa.lock); -+ -+ return ret; -+} -+ -+static const struct file_operations fops_string = { -+ .read = param_string_get, -+ .write = param_string_set, -+ .open = simple_open, -+ .llseek = default_llseek, -+}; -+ -+int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, -+ void *addr, size_t size, -+ enum kbase_ipa_model_param_type type) -+{ -+ struct kbase_ipa_model_param *param; -+ -+ param = kzalloc(sizeof(*param), GFP_KERNEL); -+ -+ if (!param) -+ return -ENOMEM; -+ -+ /* 'name' is stack-allocated for array elements, so copy it into -+ * heap-allocated storage */ -+ param->name = kstrdup(name, GFP_KERNEL); -+ param->addr.voidp = addr; -+ param->size = size; -+ param->type = type; -+ param->model = model; -+ -+ list_add(¶m->link, &model->params); -+ -+ return 0; -+} -+ -+void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) -+{ -+ struct kbase_ipa_model_param *param_p, *param_n; -+ -+ list_for_each_entry_safe(param_p, param_n, &model->params, link) { -+ list_del(¶m_p->link); -+ kfree(param_p->name); -+ kfree(param_p); -+ } -+} -+ -+static void kbase_ipa_model_debugfs_init(struct kbase_ipa_model *model) -+{ -+ struct list_head *it; -+ struct dentry *dir; -+ -+ lockdep_assert_held(&model->kbdev->ipa.lock); -+ -+ dir = debugfs_create_dir(model->ops->name, -+ model->kbdev->mali_debugfs_directory); -+ -+ if (!dir) { -+ dev_err(model->kbdev->dev, -+ "Couldn't create mali debugfs %s directory", -+ model->ops->name); -+ return; -+ } -+ -+ list_for_each(it, &model->params) { -+ struct kbase_ipa_model_param *param = -+ list_entry(it, -+ struct kbase_ipa_model_param, -+ link); -+ const struct file_operations *fops = NULL; -+ -+ switch (param->type) { -+ case PARAM_TYPE_S32: -+ fops = &fops_s32; -+ break; -+ case PARAM_TYPE_STRING: -+ fops = &fops_string; -+ break; -+ } -+ -+ if (unlikely(!fops)) { -+ dev_err(model->kbdev->dev, -+ "Type not set for %s parameter %s\n", -+ model->ops->name, param->name); -+ } else { -+ debugfs_create_file(param->name, S_IRUGO | S_IWUSR, -+ dir, param, fops); -+ } -+ } -+} -+ -+void kbase_ipa_debugfs_init(struct kbase_device *kbdev) -+{ -+ mutex_lock(&kbdev->ipa.lock); -+ -+ if (kbdev->ipa.configured_model != kbdev->ipa.fallback_model) -+ kbase_ipa_model_debugfs_init(kbdev->ipa.configured_model); -+ kbase_ipa_model_debugfs_init(kbdev->ipa.fallback_model); -+ -+ mutex_unlock(&kbdev->ipa.lock); -+} -diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h -new file mode 100755 -index 000000000..ec06e2096 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h -@@ -0,0 +1,49 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_IPA_DEBUGFS_H_ -+#define _KBASE_IPA_DEBUGFS_H_ -+ -+enum kbase_ipa_model_param_type { -+ PARAM_TYPE_S32 = 1, -+ PARAM_TYPE_STRING, -+}; -+ -+#ifdef CONFIG_DEBUG_FS -+ -+void kbase_ipa_debugfs_init(struct kbase_device *kbdev); -+int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, -+ void *addr, size_t size, -+ enum kbase_ipa_model_param_type type); -+void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model); -+ -+#else /* CONFIG_DEBUG_FS */ -+ -+static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, -+ const char *name, void *addr, -+ size_t size, -+ enum kbase_ipa_model_param_type type) -+{ -+ return 0; -+} -+ -+static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) -+{ } -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+#endif /* _KBASE_IPA_DEBUGFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c -new file mode 100755 -index 000000000..da0a4d4a0 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c -@@ -0,0 +1,222 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#ifdef CONFIG_DEVFREQ_THERMAL -+#include -+#endif -+#include -+#include -+ -+#include "mali_kbase.h" -+#include "mali_kbase_defs.h" -+ -+/* -+ * This model is primarily designed for the Juno platform. It may not be -+ * suitable for other platforms. The additional resources in this model -+ * should preferably be minimal, as this model is rarely used when a dynamic -+ * model is available. -+ */ -+ -+/** -+ * struct kbase_ipa_model_simple_data - IPA context per device -+ * @dynamic_coefficient: dynamic coefficient of the model -+ * @static_coefficient: static coefficient of the model -+ * @ts: Thermal scaling coefficients of the model -+ * @tz_name: Thermal zone name -+ * @gpu_tz: thermal zone device -+ */ -+ -+struct kbase_ipa_model_simple_data { -+ u32 dynamic_coefficient; -+ u32 static_coefficient; -+ s32 ts[4]; -+ char tz_name[16]; -+ struct thermal_zone_device *gpu_tz; -+}; -+#define FALLBACK_STATIC_TEMPERATURE 55000 -+ -+/** -+ * calculate_temp_scaling_factor() - Calculate temperature scaling coefficient -+ * @ts: Signed coefficients, in order t^0 to t^3, with units Deg^-N -+ * @t: Temperature, in mDeg C. Range: -2^17 < t < 2^17 -+ * -+ * Scale the temperature according to a cubic polynomial whose coefficients are -+ * provided in the device tree. The result is used to scale the static power -+ * coefficient, where 1000000 means no change. -+ * -+ * Return: Temperature scaling factor. Approx range 0 < ret < 10,000,000. -+ */ -+static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) -+{ -+ /* Range: -2^24 < t2 < 2^24 m(Deg^2) */ -+ u32 remainder; -+ // static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) -+ const s64 t2 = div_s64_rem((t * t), 1000, &remainder); -+ -+ /* Range: -2^31 < t3 < 2^31 m(Deg^3) */ -+ const s64 t3 = div_s64_rem((t * t2), 1000, &remainder); -+ -+ /* -+ * Sum the parts. t^[1-3] are in m(Deg^N), but the coefficients are in -+ * Deg^-N, so we need to multiply the last coefficient by 1000. -+ * Range: -2^63 < res_big < 2^63 -+ */ -+ const s64 res_big = ts[3] * t3 /* +/- 2^62 */ -+ + ts[2] * t2 /* +/- 2^55 */ -+ + ts[1] * t /* +/- 2^48 */ -+ + ts[0] * 1000; /* +/- 2^41 */ -+ -+ /* Range: -2^60 < res_unclamped < 2^60 */ -+ s64 res_unclamped = div_s64_rem(res_big, 1000, &remainder); -+ -+ /* Clamp to range of 0x to 10x the static power */ -+ return clamp(res_unclamped, (s64) 0, (s64) 10000000); -+} -+ -+static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) -+{ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) -+ unsigned long temp; -+#else -+ int temp; -+#endif -+ u32 temp_scaling_factor; -+ struct kbase_ipa_model_simple_data *model_data = -+ (struct kbase_ipa_model_simple_data *) model->model_data; -+ struct thermal_zone_device *gpu_tz = model_data->gpu_tz; -+ u64 coeffp_big; -+ -+ if (gpu_tz) { -+ int ret; -+ -+ ret = gpu_tz->ops->get_temp(gpu_tz, &temp); -+ if (ret) { -+ pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n", -+ ret); -+ temp = FALLBACK_STATIC_TEMPERATURE; -+ } -+ } else { -+ temp = FALLBACK_STATIC_TEMPERATURE; -+ } -+ -+ temp_scaling_factor = calculate_temp_scaling_factor(model_data->ts, -+ temp); -+ coeffp_big = (u64)model_data->static_coefficient * temp_scaling_factor; -+ *coeffp = div_u64(coeffp_big, 1000000); -+ -+ return 0; -+} -+ -+static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, -+ u32 current_freq) -+{ -+ struct kbase_ipa_model_simple_data *model_data = -+ (struct kbase_ipa_model_simple_data *) model->model_data; -+ -+ *coeffp = model_data->dynamic_coefficient; -+ -+ return 0; -+} -+ -+static int add_params(struct kbase_ipa_model *model) -+{ -+ int err = 0; -+ struct kbase_ipa_model_simple_data *model_data = -+ (struct kbase_ipa_model_simple_data *)model->model_data; -+ -+ err = kbase_ipa_model_add_param_s32(model, "static-coefficient", -+ &model_data->static_coefficient, -+ 1, true); -+ if (err) -+ goto end; -+ -+ err = kbase_ipa_model_add_param_s32(model, "dynamic-coefficient", -+ &model_data->dynamic_coefficient, -+ 1, true); -+ if (err) -+ goto end; -+ -+ err = kbase_ipa_model_add_param_s32(model, "ts", -+ model_data->ts, 4, true); -+ if (err) -+ goto end; -+ -+ err = kbase_ipa_model_add_param_string(model, "thermal-zone", -+ model_data->tz_name, -+ sizeof(model_data->tz_name), true); -+ -+end: -+ return err; -+} -+ -+static int kbase_simple_power_model_init(struct kbase_ipa_model *model) -+{ -+ int err; -+ struct kbase_ipa_model_simple_data *model_data; -+ -+ model_data = kzalloc(sizeof(struct kbase_ipa_model_simple_data), -+ GFP_KERNEL); -+ if (!model_data) -+ return -ENOMEM; -+ -+ model->model_data = (void *) model_data; -+ -+ err = add_params(model); -+ -+ return err; -+} -+ -+static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) -+{ -+ struct kbase_ipa_model_simple_data *model_data = -+ (struct kbase_ipa_model_simple_data *)model->model_data; -+ -+ if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { -+ model_data->gpu_tz = NULL; -+ } else { -+ model_data->gpu_tz = thermal_zone_get_zone_by_name(model_data->tz_name); -+ -+ if (IS_ERR(model_data->gpu_tz)) { -+ pr_warn_ratelimited("Error %ld getting thermal zone \'%s\', not yet ready?\n", -+ PTR_ERR(model_data->gpu_tz), -+ model_data->tz_name); -+ model_data->gpu_tz = NULL; -+ return -EPROBE_DEFER; -+ } -+ } -+ -+ return 0; -+} -+ -+static void kbase_simple_power_model_term(struct kbase_ipa_model *model) -+{ -+ struct kbase_ipa_model_simple_data *model_data = -+ (struct kbase_ipa_model_simple_data *)model->model_data; -+ -+ kfree(model_data); -+} -+ -+struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { -+ .name = "mali-simple-power-model", -+ .init = &kbase_simple_power_model_init, -+ .recalculate = &kbase_simple_power_model_recalculate, -+ .term = &kbase_simple_power_model_term, -+ .get_dynamic_coeff = &model_dynamic_coeff, -+ .get_static_coeff = &model_static_coeff, -+ .do_utilization_scaling_in_framework = true, -+}; -diff --git a/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h b/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h -new file mode 100755 -index 000000000..6be0a334f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h -@@ -0,0 +1,311 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, -+ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py -+ * For more information see base/tools/hwconfig_generator/README -+ */ -+ -+#ifndef _BASE_HWCONFIG_FEATURES_H_ -+#define _BASE_HWCONFIG_FEATURES_H_ -+ -+enum base_hw_feature { -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, -+ BASE_HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_V4, -+ BASE_HW_FEATURE_FLUSH_REDUCTION, -+ BASE_HW_FEATURE_PROTECTED_MODE, -+ BASE_HW_FEATURE_COHERENCY_REG, -+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -+ BASE_HW_FEATURE_AARCH64_MMU, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_generic[] = { -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_t60x[] = { -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_V4, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_t62x[] = { -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_V4, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_t72x[] = { -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_V4, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_t76x[] = { -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_tFxx[] = { -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_t83x[] = { -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_t82x[] = { -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_tMIx[] = { -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_FLUSH_REDUCTION, -+ BASE_HW_FEATURE_PROTECTED_MODE, -+ BASE_HW_FEATURE_COHERENCY_REG, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_tHEx[] = { -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_FLUSH_REDUCTION, -+ BASE_HW_FEATURE_PROTECTED_MODE, -+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -+ BASE_HW_FEATURE_COHERENCY_REG, -+ BASE_HW_FEATURE_END -+}; -+ -+static const enum base_hw_feature base_hw_features_tSIx[] = { -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_FLUSH_REDUCTION, -+ BASE_HW_FEATURE_PROTECTED_MODE, -+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -+ BASE_HW_FEATURE_COHERENCY_REG, -+ BASE_HW_FEATURE_END -+}; -+ -+ -+#ifdef MALI_INCLUDE_TKAX -+static const enum base_hw_feature base_hw_features_tKAx[] = { -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_FLUSH_REDUCTION, -+ BASE_HW_FEATURE_PROTECTED_MODE, -+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -+ BASE_HW_FEATURE_COHERENCY_REG, -+ BASE_HW_FEATURE_END -+}; -+ -+#endif /* MALI_INCLUDE_TKAX */ -+ -+#ifdef MALI_INCLUDE_TTRX -+static const enum base_hw_feature base_hw_features_tTRx[] = { -+ BASE_HW_FEATURE_33BIT_VA, -+ BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -+ BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -+ BASE_HW_FEATURE_XAFFINITY, -+ BASE_HW_FEATURE_WARPING, -+ BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -+ BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -+ BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -+ BASE_HW_FEATURE_BRNDOUT_CC, -+ BASE_HW_FEATURE_BRNDOUT_KILL, -+ BASE_HW_FEATURE_LD_ST_LEA_TEX, -+ BASE_HW_FEATURE_LD_ST_TILEBUFFER, -+ BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -+ BASE_HW_FEATURE_MRT, -+ BASE_HW_FEATURE_MSAA_16X, -+ BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -+ BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -+ BASE_HW_FEATURE_T7XX_PAIRING_RULES, -+ BASE_HW_FEATURE_TEST4_DATUM_MODE, -+ BASE_HW_FEATURE_FLUSH_REDUCTION, -+ BASE_HW_FEATURE_PROTECTED_MODE, -+ BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -+ BASE_HW_FEATURE_COHERENCY_REG, -+ BASE_HW_FEATURE_END -+}; -+ -+#endif /* MALI_INCLUDE_TTRX */ -+ -+#endif /* _BASE_HWCONFIG_FEATURES_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h b/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h -new file mode 100755 -index 000000000..6d7e5c57e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h -@@ -0,0 +1,1098 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, -+ * please update base/tools/hwconfig_generator/hwc_{issues,features}.py -+ * For more information see base/tools/hwconfig_generator/README -+ */ -+ -+#ifndef _BASE_HWCONFIG_ISSUES_H_ -+#define _BASE_HWCONFIG_ISSUES_H_ -+ -+enum base_hw_issue { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_6367, -+ BASE_HW_ISSUE_6398, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_6787, -+ BASE_HW_ISSUE_7027, -+ BASE_HW_ISSUE_7144, -+ BASE_HW_ISSUE_7304, -+ BASE_HW_ISSUE_8073, -+ BASE_HW_ISSUE_8186, -+ BASE_HW_ISSUE_8215, -+ BASE_HW_ISSUE_8245, -+ BASE_HW_ISSUE_8250, -+ BASE_HW_ISSUE_8260, -+ BASE_HW_ISSUE_8280, -+ BASE_HW_ISSUE_8316, -+ BASE_HW_ISSUE_8381, -+ BASE_HW_ISSUE_8394, -+ BASE_HW_ISSUE_8401, -+ BASE_HW_ISSUE_8408, -+ BASE_HW_ISSUE_8443, -+ BASE_HW_ISSUE_8456, -+ BASE_HW_ISSUE_8564, -+ BASE_HW_ISSUE_8634, -+ BASE_HW_ISSUE_8778, -+ BASE_HW_ISSUE_8791, -+ BASE_HW_ISSUE_8833, -+ BASE_HW_ISSUE_8879, -+ BASE_HW_ISSUE_8896, -+ BASE_HW_ISSUE_8975, -+ BASE_HW_ISSUE_8986, -+ BASE_HW_ISSUE_8987, -+ BASE_HW_ISSUE_9010, -+ BASE_HW_ISSUE_9418, -+ BASE_HW_ISSUE_9423, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_9510, -+ BASE_HW_ISSUE_9566, -+ BASE_HW_ISSUE_9630, -+ BASE_HW_ISSUE_10127, -+ BASE_HW_ISSUE_10327, -+ BASE_HW_ISSUE_10410, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10487, -+ BASE_HW_ISSUE_10607, -+ BASE_HW_ISSUE_10632, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10676, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10797, -+ BASE_HW_ISSUE_10817, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_10959, -+ BASE_HW_ISSUE_10969, -+ BASE_HW_ISSUE_10984, -+ BASE_HW_ISSUE_10995, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11035, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_26, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3542, -+ BASE_HW_ISSUE_T76X_3556, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_7940, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TMIX_8138, -+ BASE_HW_ISSUE_TMIX_8206, -+ BASE_HW_ISSUE_TMIX_8343, -+ BASE_HW_ISSUE_TMIX_8463, -+ BASE_HW_ISSUE_TMIX_8456, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_generic[] = { -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t60x_r0p0_15dev0[] = { -+ BASE_HW_ISSUE_6367, -+ BASE_HW_ISSUE_6398, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_6787, -+ BASE_HW_ISSUE_7027, -+ BASE_HW_ISSUE_7144, -+ BASE_HW_ISSUE_7304, -+ BASE_HW_ISSUE_8073, -+ BASE_HW_ISSUE_8186, -+ BASE_HW_ISSUE_8215, -+ BASE_HW_ISSUE_8245, -+ BASE_HW_ISSUE_8250, -+ BASE_HW_ISSUE_8260, -+ BASE_HW_ISSUE_8280, -+ BASE_HW_ISSUE_8316, -+ BASE_HW_ISSUE_8381, -+ BASE_HW_ISSUE_8394, -+ BASE_HW_ISSUE_8401, -+ BASE_HW_ISSUE_8408, -+ BASE_HW_ISSUE_8443, -+ BASE_HW_ISSUE_8456, -+ BASE_HW_ISSUE_8564, -+ BASE_HW_ISSUE_8634, -+ BASE_HW_ISSUE_8778, -+ BASE_HW_ISSUE_8791, -+ BASE_HW_ISSUE_8833, -+ BASE_HW_ISSUE_8896, -+ BASE_HW_ISSUE_8975, -+ BASE_HW_ISSUE_8986, -+ BASE_HW_ISSUE_8987, -+ BASE_HW_ISSUE_9010, -+ BASE_HW_ISSUE_9418, -+ BASE_HW_ISSUE_9423, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_9510, -+ BASE_HW_ISSUE_9566, -+ BASE_HW_ISSUE_9630, -+ BASE_HW_ISSUE_10410, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10487, -+ BASE_HW_ISSUE_10607, -+ BASE_HW_ISSUE_10632, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10676, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_10969, -+ BASE_HW_ISSUE_10984, -+ BASE_HW_ISSUE_10995, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11035, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_3964, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t60x_r0p0_eac[] = { -+ BASE_HW_ISSUE_6367, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_6787, -+ BASE_HW_ISSUE_7027, -+ BASE_HW_ISSUE_7304, -+ BASE_HW_ISSUE_8408, -+ BASE_HW_ISSUE_8564, -+ BASE_HW_ISSUE_8778, -+ BASE_HW_ISSUE_8975, -+ BASE_HW_ISSUE_9010, -+ BASE_HW_ISSUE_9418, -+ BASE_HW_ISSUE_9423, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_9510, -+ BASE_HW_ISSUE_10410, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10487, -+ BASE_HW_ISSUE_10607, -+ BASE_HW_ISSUE_10632, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10676, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_10969, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11035, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t60x_r0p1[] = { -+ BASE_HW_ISSUE_6367, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_6787, -+ BASE_HW_ISSUE_7027, -+ BASE_HW_ISSUE_7304, -+ BASE_HW_ISSUE_8408, -+ BASE_HW_ISSUE_8564, -+ BASE_HW_ISSUE_8778, -+ BASE_HW_ISSUE_8975, -+ BASE_HW_ISSUE_9010, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_9510, -+ BASE_HW_ISSUE_10410, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10487, -+ BASE_HW_ISSUE_10607, -+ BASE_HW_ISSUE_10632, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10676, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11035, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t62x_r0p1[] = { -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10127, -+ BASE_HW_ISSUE_10327, -+ BASE_HW_ISSUE_10410, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10487, -+ BASE_HW_ISSUE_10607, -+ BASE_HW_ISSUE_10632, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10676, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10817, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_10959, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11035, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t62x_r1p0[] = { -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_10959, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t62x_r1p1[] = { -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_10959, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t76x_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_26, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3542, -+ BASE_HW_ISSUE_T76X_3556, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t76x_r0p1[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_26, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3542, -+ BASE_HW_ISSUE_T76X_3556, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t76x_r0p1_50rel0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_26, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3542, -+ BASE_HW_ISSUE_T76X_3556, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t76x_r0p2[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_26, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3542, -+ BASE_HW_ISSUE_T76X_3556, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t76x_r0p3[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_26, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3542, -+ BASE_HW_ISSUE_T76X_3556, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t76x_r1p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t72x_r0p0[] = { -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10797, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t72x_r1p0[] = { -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10797, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t72x_r1p1[] = { -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10684, -+ BASE_HW_ISSUE_10797, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t72x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10471, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10797, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t76x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t60x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_8778, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t62x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_6402, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10472, -+ BASE_HW_ISSUE_10649, -+ BASE_HW_ISSUE_10931, -+ BASE_HW_ISSUE_11012, -+ BASE_HW_ISSUE_11020, -+ BASE_HW_ISSUE_11024, -+ BASE_HW_ISSUE_11042, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3964, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tFRx_r0p1[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tFRx_r0p2[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tFRx_r1p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tFRx_r2p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_tFRx[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t86x_r0p2[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t86x_r1p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t86x_r2p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3966, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t86x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t83x_r0p1[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t83x_r1p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t83x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t82x_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3964, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t82x_r0p1[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1909, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_t82x_r1p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10821, -+ BASE_HW_ISSUE_10883, -+ BASE_HW_ISSUE_10946, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T720_1386, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_T76X_3960, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_t82x[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_11051, -+ BASE_HW_ISSUE_T76X_1963, -+ BASE_HW_ISSUE_T76X_3086, -+ BASE_HW_ISSUE_T76X_3700, -+ BASE_HW_ISSUE_T76X_3793, -+ BASE_HW_ISSUE_T76X_3979, -+ BASE_HW_ISSUE_TMIX_7891, -+ GPUCORE_1619, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_T76X_3953, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TMIX_8138, -+ BASE_HW_ISSUE_TMIX_8206, -+ BASE_HW_ISSUE_TMIX_8343, -+ BASE_HW_ISSUE_TMIX_8463, -+ BASE_HW_ISSUE_TMIX_8456, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_11054, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_7940, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TMIX_8138, -+ BASE_HW_ISSUE_TMIX_8206, -+ BASE_HW_ISSUE_TMIX_8343, -+ BASE_HW_ISSUE_TMIX_8463, -+ BASE_HW_ISSUE_TMIX_8456, -+ BASE_HW_ISSUE_TMIX_8438, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_tMIx[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_7940, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TMIX_8138, -+ BASE_HW_ISSUE_TMIX_8206, -+ BASE_HW_ISSUE_TMIX_8343, -+ BASE_HW_ISSUE_TMIX_8456, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_10682, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_tHEx[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_7891, -+ BASE_HW_ISSUE_TMIX_8042, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+static const enum base_hw_issue base_hw_issues_model_tSIx[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+ -+ -+#ifdef MALI_INCLUDE_TKAX -+static const enum base_hw_issue base_hw_issues_tKAx_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+#endif /* MALI_INCLUDE_TKAX */ -+ -+#ifdef MALI_INCLUDE_TKAX -+static const enum base_hw_issue base_hw_issues_model_tKAx[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+#endif /* MALI_INCLUDE_TKAX */ -+ -+#ifdef MALI_INCLUDE_TTRX -+static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+#endif /* MALI_INCLUDE_TTRX */ -+ -+#ifdef MALI_INCLUDE_TTRX -+static const enum base_hw_issue base_hw_issues_model_tTRx[] = { -+ BASE_HW_ISSUE_5736, -+ BASE_HW_ISSUE_9435, -+ BASE_HW_ISSUE_TMIX_8133, -+ BASE_HW_ISSUE_TSIX_1116, -+ BASE_HW_ISSUE_END -+}; -+ -+#endif /* MALI_INCLUDE_TTRX */ -+ -+#endif /* _BASE_HWCONFIG_ISSUES_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_base_kernel.h b/drivers/gpu/arm/midgard/mali_base_kernel.h -new file mode 100755 -index 000000000..ea5e473ca ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_base_kernel.h -@@ -0,0 +1,1858 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file -+ * Base structures shared with the kernel. -+ */ -+ -+#ifndef _BASE_KERNEL_H_ -+#define _BASE_KERNEL_H_ -+ -+#ifndef __user -+#define __user -+#endif -+ -+/* Support UK6 IOCTLS */ -+#define BASE_LEGACY_UK6_SUPPORT 1 -+ -+/* Support UK7 IOCTLS */ -+/* NB: To support UK6 we also need to support UK7 */ -+#define BASE_LEGACY_UK7_SUPPORT 1 -+ -+/* Support UK8 IOCTLS */ -+#define BASE_LEGACY_UK8_SUPPORT 1 -+ -+/* Support UK9 IOCTLS */ -+#define BASE_LEGACY_UK9_SUPPORT 1 -+ -+/* Support UK10_2 IOCTLS */ -+#define BASE_LEGACY_UK10_2_SUPPORT 1 -+ -+/* Support UK10_4 IOCTLS */ -+#define BASE_LEGACY_UK10_4_SUPPORT 1 -+ -+typedef struct base_mem_handle { -+ struct { -+ u64 handle; -+ } basep; -+} base_mem_handle; -+ -+#include "mali_base_mem_priv.h" -+#include "mali_kbase_profiling_gator_api.h" -+#include "mali_midg_coherency.h" -+#include "mali_kbase_gpu_id.h" -+ -+/* -+ * Dependency stuff, keep it private for now. May want to expose it if -+ * we decide to make the number of semaphores a configurable -+ * option. -+ */ -+#define BASE_JD_ATOM_COUNT 512 -+ -+#define BASEP_JD_SEM_PER_WORD_LOG2 5 -+#define BASEP_JD_SEM_PER_WORD (1 << BASEP_JD_SEM_PER_WORD_LOG2) -+#define BASEP_JD_SEM_WORD_NR(x) ((x) >> BASEP_JD_SEM_PER_WORD_LOG2) -+#define BASEP_JD_SEM_MASK_IN_WORD(x) (1 << ((x) & (BASEP_JD_SEM_PER_WORD - 1))) -+#define BASEP_JD_SEM_ARRAY_SIZE BASEP_JD_SEM_WORD_NR(BASE_JD_ATOM_COUNT) -+ -+/* Set/reset values for a software event */ -+#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) -+#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) -+ -+#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 -+ -+#define BASE_MAX_COHERENT_GROUPS 16 -+ -+#if defined CDBG_ASSERT -+#define LOCAL_ASSERT CDBG_ASSERT -+#elif defined KBASE_DEBUG_ASSERT -+#define LOCAL_ASSERT KBASE_DEBUG_ASSERT -+#else -+#error assert macro not defined! -+#endif -+ -+#if defined PAGE_MASK -+#define LOCAL_PAGE_LSB ~PAGE_MASK -+#else -+#include -+ -+#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 -+#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) -+#else -+#error Failed to find page size -+#endif -+#endif -+ -+/** 32/64-bit neutral way to represent pointers */ -+typedef union kbase_pointer { -+ void __user *value; /**< client should store their pointers here */ -+ u32 compat_value; /**< 64-bit kernels should fetch value here when handling 32-bit clients */ -+ u64 sizer; /**< Force 64-bit storage for all clients regardless */ -+} kbase_pointer; -+ -+/** -+ * @addtogroup base_user_api User-side Base APIs -+ * @{ -+ */ -+ -+/** -+ * @addtogroup base_user_api_memory User-side Base Memory APIs -+ * @{ -+ */ -+ -+/** -+ * typedef base_mem_alloc_flags - Memory allocation, access/hint flags. -+ * -+ * A combination of MEM_PROT/MEM_HINT flags must be passed to each allocator -+ * in order to determine the best cache policy. Some combinations are -+ * of course invalid (e.g. MEM_PROT_CPU_WR | MEM_HINT_CPU_RD), -+ * which defines a write-only region on the CPU side, which is -+ * heavily read by the CPU... -+ * Other flags are only meaningful to a particular allocator. -+ * More flags can be added to this list, as long as they don't clash -+ * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). -+ */ -+typedef u32 base_mem_alloc_flags; -+ -+/* Memory allocation, access/hint flags. -+ * -+ * See base_mem_alloc_flags. -+ */ -+ -+/* IN */ -+/* Read access CPU side -+ */ -+#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) -+ -+/* Write access CPU side -+ */ -+#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) -+ -+/* Read access GPU side -+ */ -+#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) -+ -+/* Write access GPU side -+ */ -+#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) -+ -+/* Execute allowed on the GPU side -+ */ -+#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) -+ -+ /* BASE_MEM_HINT flags have been removed, but their values are reserved -+ * for backwards compatibility with older user-space drivers. The values -+ * can be re-used once support for r5p0 user-space drivers is removed, -+ * presumably in r7p0. -+ * -+ * RESERVED: (1U << 5) -+ * RESERVED: (1U << 6) -+ * RESERVED: (1U << 7) -+ * RESERVED: (1U << 8) -+ */ -+ -+/* Grow backing store on GPU Page Fault -+ */ -+#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) -+ -+/* Page coherence Outer shareable, if available -+ */ -+#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) -+ -+/* Page coherence Inner shareable -+ */ -+#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) -+ -+/* Should be cached on the CPU -+ */ -+#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) -+ -+/* IN/OUT */ -+/* Must have same VA on both the GPU and the CPU -+ */ -+#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) -+ -+/* OUT */ -+/* Must call mmap to acquire a GPU address for the alloc -+ */ -+#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) -+ -+/* IN */ -+/* Page coherence Outer shareable, required. -+ */ -+#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) -+ -+/* Secure memory -+ */ -+#define BASE_MEM_SECURE ((base_mem_alloc_flags)1 << 16) -+ -+/* Not needed physical memory -+ */ -+#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) -+ -+/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the -+ * addresses to be the same -+ */ -+#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) -+ -+/* Number of bits used as flags for base memory management -+ * -+ * Must be kept in sync with the base_mem_alloc_flags flags -+ */ -+#define BASE_MEM_FLAGS_NR_BITS 19 -+ -+/* A mask for all output bits, excluding IN/OUT bits. -+ */ -+#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP -+ -+/* A mask for all input bits, including IN/OUT bits. -+ */ -+#define BASE_MEM_FLAGS_INPUT_MASK \ -+ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) -+ -+/* A mask for all the flags which are modifiable via the base_mem_set_flags -+ * interface. -+ */ -+#define BASE_MEM_FLAGS_MODIFIABLE \ -+ (BASE_MEM_DONT_NEED | BASE_MEM_COHERENT_SYSTEM | \ -+ BASE_MEM_COHERENT_LOCAL) -+ -+/** -+ * enum base_mem_import_type - Memory types supported by @a base_mem_import -+ * -+ * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type -+ * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id. -+ * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) -+ * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a -+ * base_mem_import_user_buffer -+ * -+ * Each type defines what the supported handle type is. -+ * -+ * If any new type is added here ARM must be contacted -+ * to allocate a numeric value for it. -+ * Do not just add a new type without synchronizing with ARM -+ * as future releases from ARM might include other new types -+ * which could clash with your custom types. -+ */ -+typedef enum base_mem_import_type { -+ BASE_MEM_IMPORT_TYPE_INVALID = 0, -+ BASE_MEM_IMPORT_TYPE_UMP = 1, -+ BASE_MEM_IMPORT_TYPE_UMM = 2, -+ BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 -+} base_mem_import_type; -+ -+/** -+ * struct base_mem_import_user_buffer - Handle of an imported user buffer -+ * -+ * @ptr: kbase_pointer to imported user buffer -+ * @length: length of imported user buffer in bytes -+ * -+ * This structure is used to represent a handle of an imported user buffer. -+ */ -+ -+struct base_mem_import_user_buffer { -+ kbase_pointer ptr; -+ u64 length; -+}; -+ -+/** -+ * @brief Invalid memory handle. -+ * -+ * Return value from functions returning @ref base_mem_handle on error. -+ * -+ * @warning @ref base_mem_handle_new_invalid must be used instead of this macro -+ * in C++ code or other situations where compound literals cannot be used. -+ */ -+#define BASE_MEM_INVALID_HANDLE ((base_mem_handle) { {BASEP_MEM_INVALID_HANDLE} }) -+ -+/** -+ * @brief Special write-alloc memory handle. -+ * -+ * A special handle is used to represent a region where a special page is mapped -+ * with a write-alloc cache setup, typically used when the write result of the -+ * GPU isn't needed, but the GPU must write anyway. -+ * -+ * @warning @ref base_mem_handle_new_write_alloc must be used instead of this macro -+ * in C++ code or other situations where compound literals cannot be used. -+ */ -+#define BASE_MEM_WRITE_ALLOC_PAGES_HANDLE ((base_mem_handle) { {BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE} }) -+ -+#define BASEP_MEM_INVALID_HANDLE (0ull << 12) -+#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) -+#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) -+#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) -+#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) -+/* reserved handles ..-64< for future special handles */ -+#define BASE_MEM_COOKIE_BASE (64ul << 12) -+#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ -+ BASE_MEM_COOKIE_BASE) -+ -+/* Mask to detect 4GB boundary alignment */ -+#define BASE_MEM_MASK_4GB 0xfffff000UL -+ -+ -+/* Bit mask of cookies used for for memory allocation setup */ -+#define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ -+ -+ -+/** -+ * @brief Result codes of changing the size of the backing store allocated to a tmem region -+ */ -+typedef enum base_backing_threshold_status { -+ BASE_BACKING_THRESHOLD_OK = 0, /**< Resize successful */ -+ BASE_BACKING_THRESHOLD_ERROR_OOM = -2, /**< Increase failed due to an out-of-memory condition */ -+ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS = -4 /**< Invalid arguments (not tmem, illegal size request, etc.) */ -+} base_backing_threshold_status; -+ -+/** -+ * @addtogroup base_user_api_memory_defered User-side Base Defered Memory Coherency APIs -+ * @{ -+ */ -+ -+/** -+ * @brief a basic memory operation (sync-set). -+ * -+ * The content of this structure is private, and should only be used -+ * by the accessors. -+ */ -+typedef struct base_syncset { -+ struct basep_syncset basep_sset; -+} base_syncset; -+ -+/** @} end group base_user_api_memory_defered */ -+ -+/** -+ * Handle to represent imported memory object. -+ * Simple opague handle to imported memory, can't be used -+ * with anything but base_external_resource_init to bind to an atom. -+ */ -+typedef struct base_import_handle { -+ struct { -+ u64 handle; -+ } basep; -+} base_import_handle; -+ -+/** @} end group base_user_api_memory */ -+ -+/** -+ * @addtogroup base_user_api_job_dispatch User-side Base Job Dispatcher APIs -+ * @{ -+ */ -+ -+typedef int platform_fence_type; -+#define INVALID_PLATFORM_FENCE ((platform_fence_type)-1) -+ -+/** -+ * Base stream handle. -+ * -+ * References an underlying base stream object. -+ */ -+typedef struct base_stream { -+ struct { -+ int fd; -+ } basep; -+} base_stream; -+ -+/** -+ * Base fence handle. -+ * -+ * References an underlying base fence object. -+ */ -+typedef struct base_fence { -+ struct { -+ int fd; -+ int stream_fd; -+ } basep; -+} base_fence; -+ -+/** -+ * @brief Per-job data -+ * -+ * This structure is used to store per-job data, and is completely unused -+ * by the Base driver. It can be used to store things such as callback -+ * function pointer, data to handle job completion. It is guaranteed to be -+ * untouched by the Base driver. -+ */ -+typedef struct base_jd_udata { -+ u64 blob[2]; /**< per-job data array */ -+} base_jd_udata; -+ -+/** -+ * @brief Memory aliasing info -+ * -+ * Describes a memory handle to be aliased. -+ * A subset of the handle can be chosen for aliasing, given an offset and a -+ * length. -+ * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a -+ * region where a special page is mapped with a write-alloc cache setup, -+ * typically used when the write result of the GPU isn't needed, but the GPU -+ * must write anyway. -+ * -+ * Offset and length are specified in pages. -+ * Offset must be within the size of the handle. -+ * Offset+length must not overrun the size of the handle. -+ * -+ * @handle Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE -+ * @offset Offset within the handle to start aliasing from, in pages. -+ * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. -+ * @length Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE -+ * specifies the number of times the special page is needed. -+ */ -+struct base_mem_aliasing_info { -+ base_mem_handle handle; -+ u64 offset; -+ u64 length; -+}; -+ -+/** -+ * struct base_jit_alloc_info - Structure which describes a JIT allocation -+ * request. -+ * @gpu_alloc_addr: The GPU virtual address to write the JIT -+ * allocated GPU virtual address to. -+ * @va_pages: The minimum number of virtual pages required. -+ * @commit_pages: The minimum number of physical pages which -+ * should back the allocation. -+ * @extent: Granularity of physical pages to grow the -+ * allocation by during a fault. -+ * @id: Unique ID provided by the caller, this is used -+ * to pair allocation and free requests. -+ * Zero is not a valid value. -+ */ -+struct base_jit_alloc_info { -+ u64 gpu_alloc_addr; -+ u64 va_pages; -+ u64 commit_pages; -+ u64 extent; -+ u8 id; -+}; -+ -+/** -+ * @brief Job dependency type. -+ * -+ * A flags field will be inserted into the atom structure to specify whether a dependency is a data or -+ * ordering dependency (by putting it before/after 'core_req' in the structure it should be possible to add without -+ * changing the structure size). -+ * When the flag is set for a particular dependency to signal that it is an ordering only dependency then -+ * errors will not be propagated. -+ */ -+typedef u8 base_jd_dep_type; -+ -+ -+#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ -+#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ -+#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ -+ -+/** -+ * @brief Job chain hardware requirements. -+ * -+ * A job chain must specify what GPU features it needs to allow the -+ * driver to schedule the job correctly. By not specifying the -+ * correct settings can/will cause an early job termination. Multiple -+ * values can be ORed together to specify multiple requirements. -+ * Special case is ::BASE_JD_REQ_DEP, which is used to express complex -+ * dependencies, and that doesn't execute anything on the hardware. -+ */ -+typedef u32 base_jd_core_req; -+ -+/* Requirements that come from the HW */ -+ -+/** -+ * No requirement, dependency only -+ */ -+#define BASE_JD_REQ_DEP ((base_jd_core_req)0) -+ -+/** -+ * Requires fragment shaders -+ */ -+#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) -+ -+/** -+ * Requires compute shaders -+ * This covers any of the following Midgard Job types: -+ * - Vertex Shader Job -+ * - Geometry Shader Job -+ * - An actual Compute Shader Job -+ * -+ * Compare this with @ref BASE_JD_REQ_ONLY_COMPUTE, which specifies that the -+ * job is specifically just the "Compute Shader" job type, and not the "Vertex -+ * Shader" nor the "Geometry Shader" job type. -+ */ -+#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) -+#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) /**< Requires tiling */ -+#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) /**< Requires cache flushes */ -+#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) /**< Requires value writeback */ -+ -+/* SW-only requirements - the HW does not expose these as part of the job slot capabilities */ -+ -+/* Requires fragment job with AFBC encoding */ -+#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) -+ -+/** -+ * SW-only requirement: coalesce completion events. -+ * If this bit is set then completion of this atom will not cause an event to -+ * be sent to userspace, whether successful or not; completion events will be -+ * deferred until an atom completes which does not have this bit set. -+ * -+ * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. -+ */ -+#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) -+ -+/** -+ * SW Only requirement: the job chain requires a coherent core group. We don't -+ * mind which coherent core group is used. -+ */ -+#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) -+ -+/** -+ * SW Only requirement: The performance counters should be enabled only when -+ * they are needed, to reduce power consumption. -+ */ -+ -+#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) -+ -+/** -+ * SW Only requirement: External resources are referenced by this atom. -+ * When external resources are referenced no syncsets can be bundled with the atom -+ * but should instead be part of a NULL jobs inserted into the dependency tree. -+ * The first pre_dep object must be configured for the external resouces to use, -+ * the second pre_dep object can be used to create other dependencies. -+ * -+ * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE. -+ */ -+#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) -+ -+/** -+ * SW Only requirement: Software defined job. Jobs with this bit set will not be submitted -+ * to the hardware but will cause some action to happen within the driver -+ */ -+#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) -+ -+#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) -+#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) -+#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) -+ -+/** -+ * SW Only requirement : Replay job. -+ * -+ * If the preceding job fails, the replay job will cause the jobs specified in -+ * the list of base_jd_replay_payload pointed to by the jc pointer to be -+ * replayed. -+ * -+ * A replay job will only cause jobs to be replayed up to BASEP_JD_REPLAY_LIMIT -+ * times. If a job fails more than BASEP_JD_REPLAY_LIMIT times then the replay -+ * job is failed, as well as any following dependencies. -+ * -+ * The replayed jobs will require a number of atom IDs. If there are not enough -+ * free atom IDs then the replay job will fail. -+ * -+ * If the preceding job does not fail, then the replay job is returned as -+ * completed. -+ * -+ * The replayed jobs will never be returned to userspace. The preceding failed -+ * job will be returned to userspace as failed; the status of this job should -+ * be ignored. Completion should be determined by the status of the replay soft -+ * job. -+ * -+ * In order for the jobs to be replayed, the job headers will have to be -+ * modified. The Status field will be reset to NOT_STARTED. If the Job Type -+ * field indicates a Vertex Shader Job then it will be changed to Null Job. -+ * -+ * The replayed jobs have the following assumptions : -+ * -+ * - No external resources. Any required external resources will be held by the -+ * replay atom. -+ * - Pre-dependencies are created based on job order. -+ * - Atom numbers are automatically assigned. -+ * - device_nr is set to 0. This is not relevant as -+ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. -+ * - Priority is inherited from the replay job. -+ */ -+#define BASE_JD_REQ_SOFT_REPLAY (BASE_JD_REQ_SOFT_JOB | 0x4) -+/** -+ * SW only requirement: event wait/trigger job. -+ * -+ * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. -+ * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the -+ * other waiting jobs. It completes immediately. -+ * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it -+ * possible for other jobs to wait upon. It completes immediately. -+ */ -+#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) -+#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) -+#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) -+ -+#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) -+ -+/** -+ * SW only requirement: Just In Time allocation -+ * -+ * This job requests a JIT allocation based on the request in the -+ * @base_jit_alloc_info structure which is passed via the jc element of -+ * the atom. -+ * -+ * It should be noted that the id entry in @base_jit_alloc_info must not -+ * be reused until it has been released via @BASE_JD_REQ_SOFT_JIT_FREE. -+ * -+ * Should this soft job fail it is expected that a @BASE_JD_REQ_SOFT_JIT_FREE -+ * soft job to free the JIT allocation is still made. -+ * -+ * The job will complete immediately. -+ */ -+#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) -+/** -+ * SW only requirement: Just In Time free -+ * -+ * This job requests a JIT allocation created by @BASE_JD_REQ_SOFT_JIT_ALLOC -+ * to be freed. The ID of the JIT allocation is passed via the jc element of -+ * the atom. -+ * -+ * The job will complete immediately. -+ */ -+#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) -+ -+/** -+ * SW only requirement: Map external resource -+ * -+ * This job requests external resource(s) are mapped once the dependencies -+ * of the job have been satisfied. The list of external resources are -+ * passed via the jc element of the atom which is a pointer to a -+ * @base_external_resource_list. -+ */ -+#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) -+/** -+ * SW only requirement: Unmap external resource -+ * -+ * This job requests external resource(s) are unmapped once the dependencies -+ * of the job has been satisfied. The list of external resources are -+ * passed via the jc element of the atom which is a pointer to a -+ * @base_external_resource_list. -+ */ -+#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) -+ -+/** -+ * HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) -+ * -+ * This indicates that the Job Chain contains Midgard Jobs of the 'Compute Shaders' type. -+ * -+ * In contrast to @ref BASE_JD_REQ_CS, this does \b not indicate that the Job -+ * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. -+ */ -+#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) -+ -+/** -+ * HW Requirement: Use the base_jd_atom::device_nr field to specify a -+ * particular core group -+ * -+ * If both @ref BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag takes priority -+ * -+ * This is only guaranteed to work for @ref BASE_JD_REQ_ONLY_COMPUTE atoms. -+ * -+ * If the core availability policy is keeping the required core group turned off, then -+ * the job will fail with a @ref BASE_JD_EVENT_PM_EVENT error code. -+ */ -+#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) -+ -+/** -+ * SW Flag: If this bit is set then the successful completion of this atom -+ * will not cause an event to be sent to userspace -+ */ -+#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) -+ -+/** -+ * SW Flag: If this bit is set then completion of this atom will not cause an -+ * event to be sent to userspace, whether successful or not. -+ */ -+#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) -+ -+/** -+ * SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. -+ * -+ * If this bit is set then the GPU's cache will not be cleaned and invalidated -+ * until a GPU job starts which does not have this bit set or a job completes -+ * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use if -+ * the CPU may have written to memory addressed by the job since the last job -+ * without this bit set was submitted. -+ */ -+#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) -+ -+/** -+ * SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. -+ * -+ * If this bit is set then the GPU's cache will not be cleaned and invalidated -+ * until a GPU job completes which does not have this bit set or a job starts -+ * which does not have the @ref BASE_JD_REQ_SKIP_CACHE_START bti set. Do not use if -+ * the CPU may read from or partially overwrite memory addressed by the job -+ * before the next job without this bit set completes. -+ */ -+#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) -+ -+/** -+ * These requirement bits are currently unused in base_jd_core_req -+ */ -+#define BASEP_JD_REQ_RESERVED \ -+ (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ -+ BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ -+ BASE_JD_REQ_EVENT_COALESCE | \ -+ BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | \ -+ BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ -+ BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END)) -+ -+/** -+ * Mask of all bits in base_jd_core_req that control the type of the atom. -+ * -+ * This allows dependency only atoms to have flags set -+ */ -+#define BASE_JD_REQ_ATOM_TYPE \ -+ (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | \ -+ BASE_JD_REQ_V | BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) -+ -+/** -+ * Mask of all bits in base_jd_core_req that control the type of a soft job. -+ */ -+#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) -+ -+/* -+ * Returns non-zero value if core requirements passed define a soft job or -+ * a dependency only job. -+ */ -+#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ -+ ((core_req & BASE_JD_REQ_SOFT_JOB) || \ -+ (core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) -+ -+/** -+ * @brief States to model state machine processed by kbasep_js_job_check_ref_cores(), which -+ * handles retaining cores for power management and affinity management. -+ * -+ * The state @ref KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY prevents an attack -+ * where lots of atoms could be submitted before powerup, and each has an -+ * affinity chosen that causes other atoms to have an affinity -+ * violation. Whilst the affinity was not causing violations at the time it -+ * was chosen, it could cause violations thereafter. For example, 1000 jobs -+ * could have had their affinity chosen during the powerup time, so any of -+ * those 1000 jobs could cause an affinity violation later on. -+ * -+ * The attack would otherwise occur because other atoms/contexts have to wait for: -+ * -# the currently running atoms (which are causing the violation) to -+ * finish -+ * -# and, the atoms that had their affinity chosen during powerup to -+ * finish. These are run preferentially because they don't cause a -+ * violation, but instead continue to cause the violation in others. -+ * -# or, the attacker is scheduled out (which might not happen for just 2 -+ * contexts) -+ * -+ * By re-choosing the affinity (which is designed to avoid violations at the -+ * time it's chosen), we break condition (2) of the wait, which minimizes the -+ * problem to just waiting for current jobs to finish (which can be bounded if -+ * the Job Scheduling Policy has a timer). -+ */ -+enum kbase_atom_coreref_state { -+ /** Starting state: No affinity chosen, and cores must be requested. kbase_jd_atom::affinity==0 */ -+ KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED, -+ /** Cores requested, but waiting for them to be powered. Requested cores given by kbase_jd_atom::affinity */ -+ KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES, -+ /** Cores given by kbase_jd_atom::affinity are powered, but affinity might be out-of-date, so must recheck */ -+ KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY, -+ /** Cores given by kbase_jd_atom::affinity are powered, and affinity is up-to-date, but must check for violations */ -+ KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS, -+ /** Cores are powered, kbase_jd_atom::affinity up-to-date, no affinity violations: atom can be submitted to HW */ -+ KBASE_ATOM_COREREF_STATE_READY -+}; -+ -+/* -+ * Base Atom priority -+ * -+ * Only certain priority levels are actually implemented, as specified by the -+ * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority -+ * level that is not one of those defined below. -+ * -+ * Priority levels only affect scheduling between atoms of the same type within -+ * a base context, and only after the atoms have had dependencies resolved. -+ * Fragment atoms does not affect non-frament atoms with lower priorities, and -+ * the other way around. For example, a low priority atom that has had its -+ * dependencies resolved might run before a higher priority atom that has not -+ * had its dependencies resolved. -+ * -+ * The scheduling between base contexts/processes and between atoms from -+ * different base contexts/processes is unaffected by atom priority. -+ * -+ * The atoms are scheduled as follows with respect to their priorities: -+ * - Let atoms 'X' and 'Y' be for the same job slot who have dependencies -+ * resolved, and atom 'X' has a higher priority than atom 'Y' -+ * - If atom 'Y' is currently running on the HW, then it is interrupted to -+ * allow atom 'X' to run soon after -+ * - If instead neither atom 'Y' nor atom 'X' are running, then when choosing -+ * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' -+ * - Any two atoms that have the same priority could run in any order with -+ * respect to each other. That is, there is no ordering constraint between -+ * atoms of the same priority. -+ */ -+typedef u8 base_jd_prio; -+ -+/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ -+#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) -+/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and -+ * BASE_JD_PRIO_LOW */ -+#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) -+/* Low atom priority. */ -+#define BASE_JD_PRIO_LOW ((base_jd_prio)2) -+ -+/* Count of the number of priority levels. This itself is not a valid -+ * base_jd_prio setting */ -+#define BASE_JD_NR_PRIO_LEVELS 3 -+ -+enum kbase_jd_atom_state { -+ /** Atom is not used */ -+ KBASE_JD_ATOM_STATE_UNUSED, -+ /** Atom is queued in JD */ -+ KBASE_JD_ATOM_STATE_QUEUED, -+ /** Atom has been given to JS (is runnable/running) */ -+ KBASE_JD_ATOM_STATE_IN_JS, -+ /** Atom has been completed, but not yet handed back to job dispatcher -+ * for dependency resolution */ -+ KBASE_JD_ATOM_STATE_HW_COMPLETED, -+ /** Atom has been completed, but not yet handed back to userspace */ -+ KBASE_JD_ATOM_STATE_COMPLETED -+}; -+ -+typedef u16 base_atom_id; /**< Type big enough to store an atom number in */ -+ -+struct base_dependency { -+ base_atom_id atom_id; /**< An atom number */ -+ base_jd_dep_type dependency_type; /**< Dependency type */ -+}; -+ -+/* This structure has changed since UK 10.2 for which base_jd_core_req was a u16 value. -+ * In order to keep the size of the structure same, padding field has been adjusted -+ * accordingly and core_req field of a u32 type (to which UK 10.3 base_jd_core_req defines) -+ * is added at the end of the structure. Place in the structure previously occupied by u16 core_req -+ * is kept but renamed to compat_core_req and as such it can be used in ioctl call for job submission -+ * as long as UK 10.2 legacy is supported. Once when this support ends, this field can be left -+ * for possible future use. */ -+typedef struct base_jd_atom_v2 { -+ u64 jc; /**< job-chain GPU address */ -+ struct base_jd_udata udata; /**< user data */ -+ kbase_pointer extres_list; /**< list of external resources */ -+ u16 nr_extres; /**< nr of external resources */ -+ u16 compat_core_req; /**< core requirements which correspond to the legacy support for UK 10.2 */ -+ struct base_dependency pre_dep[2]; /**< pre-dependencies, one need to use SETTER function to assign this field, -+ this is done in order to reduce possibility of improper assigment of a dependency field */ -+ base_atom_id atom_number; /**< unique number to identify the atom */ -+ base_jd_prio prio; /**< Atom priority. Refer to @ref base_jd_prio for more details */ -+ u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ -+ u8 padding[1]; -+ base_jd_core_req core_req; /**< core requirements */ -+} base_jd_atom_v2; -+ -+#ifdef BASE_LEGACY_UK6_SUPPORT -+struct base_jd_atom_v2_uk6 { -+ u64 jc; /**< job-chain GPU address */ -+ struct base_jd_udata udata; /**< user data */ -+ kbase_pointer extres_list; /**< list of external resources */ -+ u16 nr_extres; /**< nr of external resources */ -+ u16 core_req; /**< core requirements */ -+ base_atom_id pre_dep[2]; /**< pre-dependencies */ -+ base_atom_id atom_number; /**< unique number to identify the atom */ -+ base_jd_prio prio; /**< priority - smaller is higher priority */ -+ u8 device_nr; /**< coregroup when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified */ -+ u8 padding[7]; -+}; -+#endif /* BASE_LEGACY_UK6_SUPPORT */ -+ -+typedef enum base_external_resource_access { -+ BASE_EXT_RES_ACCESS_SHARED, -+ BASE_EXT_RES_ACCESS_EXCLUSIVE -+} base_external_resource_access; -+ -+typedef struct base_external_resource { -+ u64 ext_resource; -+} base_external_resource; -+ -+ -+/** -+ * The maximum number of external resources which can be mapped/unmapped -+ * in a single request. -+ */ -+#define BASE_EXT_RES_COUNT_MAX 10 -+ -+/** -+ * struct base_external_resource_list - Structure which describes a list of -+ * external resources. -+ * @count: The number of resources. -+ * @ext_res: Array of external resources which is -+ * sized at allocation time. -+ */ -+struct base_external_resource_list { -+ u64 count; -+ struct base_external_resource ext_res[1]; -+}; -+ -+struct base_jd_debug_copy_buffer { -+ u64 address; -+ u64 size; -+ struct base_external_resource extres; -+}; -+ -+/** -+ * @brief Setter for a dependency structure -+ * -+ * @param[in] dep The kbase jd atom dependency to be initialized. -+ * @param id The atom_id to be assigned. -+ * @param dep_type The dep_type to be assigned. -+ * -+ */ -+static inline void base_jd_atom_dep_set(struct base_dependency *dep, -+ base_atom_id id, base_jd_dep_type dep_type) -+{ -+ LOCAL_ASSERT(dep != NULL); -+ -+ /* -+ * make sure we don't set not allowed combinations -+ * of atom_id/dependency_type. -+ */ -+ LOCAL_ASSERT((id == 0 && dep_type == BASE_JD_DEP_TYPE_INVALID) || -+ (id > 0 && dep_type != BASE_JD_DEP_TYPE_INVALID)); -+ -+ dep->atom_id = id; -+ dep->dependency_type = dep_type; -+} -+ -+/** -+ * @brief Make a copy of a dependency structure -+ * -+ * @param[in,out] dep The kbase jd atom dependency to be written. -+ * @param[in] from The dependency to make a copy from. -+ * -+ */ -+static inline void base_jd_atom_dep_copy(struct base_dependency *dep, -+ const struct base_dependency *from) -+{ -+ LOCAL_ASSERT(dep != NULL); -+ -+ base_jd_atom_dep_set(dep, from->atom_id, from->dependency_type); -+} -+ -+/** -+ * @brief Soft-atom fence trigger setup. -+ * -+ * Sets up an atom to be a SW-only atom signaling a fence -+ * when it reaches the run state. -+ * -+ * Using the existing base dependency system the fence can -+ * be set to trigger when a GPU job has finished. -+ * -+ * The base fence object must not be terminated until the atom -+ * has been submitted to @a base_jd_submit and @a base_jd_submit has returned. -+ * -+ * @a fence must be a valid fence set up with @a base_fence_init. -+ * Calling this function with a uninitialized fence results in undefined behavior. -+ * -+ * @param[out] atom A pre-allocated atom to configure as a fence trigger SW atom -+ * @param[in] fence The base fence object to trigger. -+ */ -+static inline void base_jd_fence_trigger_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) -+{ -+ LOCAL_ASSERT(atom); -+ LOCAL_ASSERT(fence); -+ LOCAL_ASSERT(fence->basep.fd == INVALID_PLATFORM_FENCE); -+ LOCAL_ASSERT(fence->basep.stream_fd >= 0); -+ atom->jc = (uintptr_t) fence; -+ atom->core_req = BASE_JD_REQ_SOFT_FENCE_TRIGGER; -+} -+ -+/** -+ * @brief Soft-atom fence wait setup. -+ * -+ * Sets up an atom to be a SW-only atom waiting on a fence. -+ * When the fence becomes triggered the atom becomes runnable -+ * and completes immediately. -+ * -+ * Using the existing base dependency system the fence can -+ * be set to block a GPU job until it has been triggered. -+ * -+ * The base fence object must not be terminated until the atom -+ * has been submitted to @a base_jd_submit and @a base_jd_submit has returned. -+ * -+ * @a fence must be a valid fence set up with @a base_fence_init or @a base_fence_import. -+ * Calling this function with a uninitialized fence results in undefined behavior. -+ * -+ * @param[out] atom A pre-allocated atom to configure as a fence wait SW atom -+ * @param[in] fence The base fence object to wait on -+ */ -+static inline void base_jd_fence_wait_setup_v2(struct base_jd_atom_v2 *atom, struct base_fence *fence) -+{ -+ LOCAL_ASSERT(atom); -+ LOCAL_ASSERT(fence); -+ LOCAL_ASSERT(fence->basep.fd >= 0); -+ atom->jc = (uintptr_t) fence; -+ atom->core_req = BASE_JD_REQ_SOFT_FENCE_WAIT; -+} -+ -+/** -+ * @brief External resource info initialization. -+ * -+ * Sets up an external resource object to reference -+ * a memory allocation and the type of access requested. -+ * -+ * @param[in] res The resource object to initialize -+ * @param handle The handle to the imported memory object, must be -+ * obtained by calling @ref base_mem_as_import_handle(). -+ * @param access The type of access requested -+ */ -+static inline void base_external_resource_init(struct base_external_resource *res, struct base_import_handle handle, base_external_resource_access access) -+{ -+ u64 address; -+ -+ address = handle.basep.handle; -+ -+ LOCAL_ASSERT(res != NULL); -+ LOCAL_ASSERT(0 == (address & LOCAL_PAGE_LSB)); -+ LOCAL_ASSERT(access == BASE_EXT_RES_ACCESS_SHARED || access == BASE_EXT_RES_ACCESS_EXCLUSIVE); -+ -+ res->ext_resource = address | (access & LOCAL_PAGE_LSB); -+} -+ -+/** -+ * @brief Job chain event code bits -+ * Defines the bits used to create ::base_jd_event_code -+ */ -+enum { -+ BASE_JD_SW_EVENT_KERNEL = (1u << 15), /**< Kernel side event */ -+ BASE_JD_SW_EVENT = (1u << 14), /**< SW defined event */ -+ BASE_JD_SW_EVENT_SUCCESS = (1u << 13), /**< Event idicates success (SW events only) */ -+ BASE_JD_SW_EVENT_JOB = (0u << 11), /**< Job related event */ -+ BASE_JD_SW_EVENT_BAG = (1u << 11), /**< Bag related event */ -+ BASE_JD_SW_EVENT_INFO = (2u << 11), /**< Misc/info event */ -+ BASE_JD_SW_EVENT_RESERVED = (3u << 11), /**< Reserved event type */ -+ BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) /**< Mask to extract the type from an event code */ -+}; -+ -+/** -+ * @brief Job chain event codes -+ * -+ * HW and low-level SW events are represented by event codes. -+ * The status of jobs which succeeded are also represented by -+ * an event code (see ::BASE_JD_EVENT_DONE). -+ * Events are usually reported as part of a ::base_jd_event. -+ * -+ * The event codes are encoded in the following way: -+ * @li 10:0 - subtype -+ * @li 12:11 - type -+ * @li 13 - SW success (only valid if the SW bit is set) -+ * @li 14 - SW event (HW event if not set) -+ * @li 15 - Kernel event (should never be seen in userspace) -+ * -+ * Events are split up into ranges as follows: -+ * - BASE_JD_EVENT_RANGE_\_START -+ * - BASE_JD_EVENT_RANGE_\_END -+ * -+ * \a code is in \'s range when: -+ * - BASE_JD_EVENT_RANGE_\_START <= code < BASE_JD_EVENT_RANGE_\_END -+ * -+ * Ranges can be asserted for adjacency by testing that the END of the previous -+ * is equal to the START of the next. This is useful for optimizing some tests -+ * for range. -+ * -+ * A limitation is that the last member of this enum must explicitly be handled -+ * (with an assert-unreachable statement) in switch statements that use -+ * variables of this type. Otherwise, the compiler warns that we have not -+ * handled that enum value. -+ */ -+typedef enum base_jd_event_code { -+ /* HW defined exceptions */ -+ -+ /** Start of HW Non-fault status codes -+ * -+ * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, -+ * because the job was hard-stopped -+ */ -+ BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, -+ -+ /* non-fatal exceptions */ -+ BASE_JD_EVENT_NOT_STARTED = 0x00, /**< Can't be seen by userspace, treated as 'previous job done' */ -+ BASE_JD_EVENT_DONE = 0x01, -+ BASE_JD_EVENT_STOPPED = 0x03, /**< Can't be seen by userspace, becomes TERMINATED, DONE or JOB_CANCELLED */ -+ BASE_JD_EVENT_TERMINATED = 0x04, /**< This is actually a fault status code - the job was hard stopped */ -+ BASE_JD_EVENT_ACTIVE = 0x08, /**< Can't be seen by userspace, jobs only returned on complete/fail/cancel */ -+ -+ /** End of HW Non-fault status codes -+ * -+ * @note Obscurely, BASE_JD_EVENT_TERMINATED indicates a real fault, -+ * because the job was hard-stopped -+ */ -+ BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, -+ -+ /** Start of HW fault and SW Error status codes */ -+ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, -+ -+ /* job exceptions */ -+ BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, -+ BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, -+ BASE_JD_EVENT_JOB_READ_FAULT = 0x42, -+ BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, -+ BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, -+ BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, -+ BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, -+ BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, -+ BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, -+ BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, -+ BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, -+ BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, -+ BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, -+ BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, -+ BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, -+ BASE_JD_EVENT_STATE_FAULT = 0x5A, -+ BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, -+ BASE_JD_EVENT_UNKNOWN = 0x7F, -+ -+ /* GPU exceptions */ -+ BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, -+ BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, -+ -+ /* MMU exceptions */ -+ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, -+ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, -+ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, -+ BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, -+ BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, -+ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, -+ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, -+ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, -+ BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, -+ BASE_JD_EVENT_ACCESS_FLAG = 0xD8, -+ -+ /* SW defined exceptions */ -+ BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, -+ BASE_JD_EVENT_TIMED_OUT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x001, -+ BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, -+ BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, -+ BASE_JD_EVENT_PM_EVENT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x004, -+ BASE_JD_EVENT_FORCE_REPLAY = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x005, -+ -+ BASE_JD_EVENT_BAG_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_BAG | 0x003, -+ -+ /** End of HW fault and SW Error status codes */ -+ BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_RESERVED | 0x3FF, -+ -+ /** Start of SW Success status codes */ -+ BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, -+ -+ BASE_JD_EVENT_PROGRESS_REPORT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_JOB | 0x000, -+ BASE_JD_EVENT_BAG_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_BAG | 0x000, -+ BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_INFO | 0x000, -+ -+ /** End of SW Success status codes */ -+ BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | BASE_JD_SW_EVENT_RESERVED | 0x3FF, -+ -+ /** Start of Kernel-only status codes. Such codes are never returned to user-space */ -+ BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, -+ BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, -+ -+ /** End of Kernel-only status codes. */ -+ BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF -+} base_jd_event_code; -+ -+/** -+ * @brief Event reporting structure -+ * -+ * This structure is used by the kernel driver to report information -+ * about GPU events. The can either be HW-specific events or low-level -+ * SW events, such as job-chain completion. -+ * -+ * The event code contains an event type field which can be extracted -+ * by ANDing with ::BASE_JD_SW_EVENT_TYPE_MASK. -+ * -+ * Based on the event type base_jd_event::data holds: -+ * @li ::BASE_JD_SW_EVENT_JOB : the offset in the ring-buffer for the completed -+ * job-chain -+ * @li ::BASE_JD_SW_EVENT_BAG : The address of the ::base_jd_bag that has -+ * been completed (ie all contained job-chains have been completed). -+ * @li ::BASE_JD_SW_EVENT_INFO : base_jd_event::data not used -+ */ -+typedef struct base_jd_event_v2 { -+ base_jd_event_code event_code; /**< event code */ -+ base_atom_id atom_number; /**< the atom number that has completed */ -+ struct base_jd_udata udata; /**< user data */ -+} base_jd_event_v2; -+ -+/** -+ * Padding required to ensure that the @ref struct base_dump_cpu_gpu_counters structure fills -+ * a full cache line. -+ */ -+ -+#define BASE_CPU_GPU_CACHE_LINE_PADDING (36) -+ -+ -+/** -+ * @brief Structure for BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS jobs. -+ * -+ * This structure is stored into the memory pointed to by the @c jc field of @ref base_jd_atom. -+ * -+ * This structure must be padded to ensure that it will occupy whole cache lines. This is to avoid -+ * cases where access to pages containing the structure is shared between cached and un-cached -+ * memory regions, which would cause memory corruption. Here we set the structure size to be 64 bytes -+ * which is the cache line for ARM A15 processors. -+ */ -+ -+typedef struct base_dump_cpu_gpu_counters { -+ u64 system_time; -+ u64 cycle_counter; -+ u64 sec; -+ u32 usec; -+ u8 padding[BASE_CPU_GPU_CACHE_LINE_PADDING]; -+} base_dump_cpu_gpu_counters; -+ -+ -+ -+/** @} end group base_user_api_job_dispatch */ -+ -+#define GPU_MAX_JOB_SLOTS 16 -+ -+/** -+ * @page page_base_user_api_gpuprops User-side Base GPU Property Query API -+ * -+ * The User-side Base GPU Property Query API encapsulates two -+ * sub-modules: -+ * -+ * - @ref base_user_api_gpuprops_dyn "Dynamic GPU Properties" -+ * - @ref base_plat_config_gpuprops "Base Platform Config GPU Properties" -+ * -+ * There is a related third module outside of Base, which is owned by the MIDG -+ * module: -+ * - @ref gpu_props_static "Midgard Compile-time GPU Properties" -+ * -+ * Base only deals with properties that vary between different Midgard -+ * implementations - the Dynamic GPU properties and the Platform Config -+ * properties. -+ * -+ * For properties that are constant for the Midgard Architecture, refer to the -+ * MIDG module. However, we will discuss their relevance here just to -+ * provide background information. -+ * -+ * @section sec_base_user_api_gpuprops_about About the GPU Properties in Base and MIDG modules -+ * -+ * The compile-time properties (Platform Config, Midgard Compile-time -+ * properties) are exposed as pre-processor macros. -+ * -+ * Complementing the compile-time properties are the Dynamic GPU -+ * Properties, which act as a conduit for the Midgard Configuration -+ * Discovery. -+ * -+ * In general, the dynamic properties are present to verify that the platform -+ * has been configured correctly with the right set of Platform Config -+ * Compile-time Properties. -+ * -+ * As a consistent guide across the entire DDK, the choice for dynamic or -+ * compile-time should consider the following, in order: -+ * -# Can the code be written so that it doesn't need to know the -+ * implementation limits at all? -+ * -# If you need the limits, get the information from the Dynamic Property -+ * lookup. This should be done once as you fetch the context, and then cached -+ * as part of the context data structure, so it's cheap to access. -+ * -# If there's a clear and arguable inefficiency in using Dynamic Properties, -+ * then use a Compile-Time Property (Platform Config, or Midgard Compile-time -+ * property). Examples of where this might be sensible follow: -+ * - Part of a critical inner-loop -+ * - Frequent re-use throughout the driver, causing significant extra load -+ * instructions or control flow that would be worthwhile optimizing out. -+ * -+ * We cannot provide an exhaustive set of examples, neither can we provide a -+ * rule for every possible situation. Use common sense, and think about: what -+ * the rest of the driver will be doing; how the compiler might represent the -+ * value if it is a compile-time constant; whether an OEM shipping multiple -+ * devices would benefit much more from a single DDK binary, instead of -+ * insignificant micro-optimizations. -+ * -+ * @section sec_base_user_api_gpuprops_dyn Dynamic GPU Properties -+ * -+ * Dynamic GPU properties are presented in two sets: -+ * -# the commonly used properties in @ref base_gpu_props, which have been -+ * unpacked from GPU register bitfields. -+ * -# The full set of raw, unprocessed properties in @ref gpu_raw_gpu_props -+ * (also a member of @ref base_gpu_props). All of these are presented in -+ * the packed form, as presented by the GPU registers themselves. -+ * -+ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to -+ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device -+ * behaving differently?". In this case, all information about the -+ * configuration is potentially useful, but it does not need to be processed -+ * by the driver. Instead, the raw registers can be processed by the Mali -+ * Tools software on the host PC. -+ * -+ * The properties returned extend the Midgard Configuration Discovery -+ * registers. For example, GPU clock speed is not specified in the Midgard -+ * Architecture, but is necessary for OpenCL's clGetDeviceInfo() function. -+ * -+ * The GPU properties are obtained by a call to -+ * _mali_base_get_gpu_props(). This simply returns a pointer to a const -+ * base_gpu_props structure. It is constant for the life of a base -+ * context. Multiple calls to _mali_base_get_gpu_props() to a base context -+ * return the same pointer to a constant structure. This avoids cache pollution -+ * of the common data. -+ * -+ * This pointer must not be freed, because it does not point to the start of a -+ * region allocated by the memory allocator; instead, just close the @ref -+ * base_context. -+ * -+ * -+ * @section sec_base_user_api_gpuprops_config Platform Config Compile-time Properties -+ * -+ * The Platform Config File sets up gpu properties that are specific to a -+ * certain platform. Properties that are 'Implementation Defined' in the -+ * Midgard Architecture spec are placed here. -+ * -+ * @note Reference configurations are provided for Midgard Implementations, such as -+ * the Mali-T600 family. The customer need not repeat this information, and can select one of -+ * these reference configurations. For example, VA_BITS, PA_BITS and the -+ * maximum number of samples per pixel might vary between Midgard Implementations, but -+ * \b not for platforms using the Mali-T604. This information is placed in -+ * the reference configuration files. -+ * -+ * The System Integrator creates the following structure: -+ * - platform_XYZ -+ * - platform_XYZ/plat -+ * - platform_XYZ/plat/plat_config.h -+ * -+ * They then edit plat_config.h, using the example plat_config.h files as a -+ * guide. -+ * -+ * At the very least, the customer must set @ref CONFIG_GPU_CORE_TYPE, and will -+ * receive a helpful \#error message if they do not do this correctly. This -+ * selects the Reference Configuration for the Midgard Implementation. The rationale -+ * behind this decision (against asking the customer to write \#include -+ * in their plat_config.h) is as follows: -+ * - This mechanism 'looks' like a regular config file (such as Linux's -+ * .config) -+ * - It is difficult to get wrong in a way that will produce strange build -+ * errors: -+ * - They need not know where the mali_t600.h, other_midg_gpu.h etc. files are stored - and -+ * so they won't accidentally pick another file with 'mali_t600' in its name -+ * - When the build doesn't work, the System Integrator may think the DDK is -+ * doesn't work, and attempt to fix it themselves: -+ * - For the @ref CONFIG_GPU_CORE_TYPE mechanism, the only way to get past the -+ * error is to set @ref CONFIG_GPU_CORE_TYPE, and this is what the \#error tells -+ * you. -+ * - For a \#include mechanism, checks must still be made elsewhere, which the -+ * System Integrator may try working around by setting \#defines (such as -+ * VA_BITS) themselves in their plat_config.h. In the worst case, they may -+ * set the prevention-mechanism \#define of -+ * "A_CORRECT_MIDGARD_CORE_WAS_CHOSEN". -+ * - In this case, they would believe they are on the right track, because -+ * the build progresses with their fix, but with errors elsewhere. -+ * -+ * However, there is nothing to prevent the customer using \#include to organize -+ * their own configurations files hierarchically. -+ * -+ * The mechanism for the header file processing is as follows: -+ * -+ * @dot -+ digraph plat_config_mechanism { -+ rankdir=BT -+ size="6,6" -+ -+ "mali_base.h"; -+ "gpu/mali_gpu.h"; -+ -+ node [ shape=box ]; -+ { -+ rank = same; ordering = out; -+ -+ "gpu/mali_gpu_props.h"; -+ "base/midg_gpus/mali_t600.h"; -+ "base/midg_gpus/other_midg_gpu.h"; -+ } -+ { rank = same; "plat/plat_config.h"; } -+ { -+ rank = same; -+ "gpu/mali_gpu.h" [ shape=box ]; -+ gpu_chooser [ label="" style="invisible" width=0 height=0 fixedsize=true ]; -+ select_gpu [ label="Mali-T600 | Other\n(select_gpu.h)" shape=polygon,sides=4,distortion=0.25 width=3.3 height=0.99 fixedsize=true ] ; -+ } -+ node [ shape=box ]; -+ { rank = same; "plat/plat_config.h"; } -+ { rank = same; "mali_base.h"; } -+ -+ "mali_base.h" -> "gpu/mali_gpu.h" -> "gpu/mali_gpu_props.h"; -+ "mali_base.h" -> "plat/plat_config.h" ; -+ "mali_base.h" -> select_gpu ; -+ -+ "plat/plat_config.h" -> gpu_chooser [style="dotted,bold" dir=none weight=4] ; -+ gpu_chooser -> select_gpu [style="dotted,bold"] ; -+ -+ select_gpu -> "base/midg_gpus/mali_t600.h" ; -+ select_gpu -> "base/midg_gpus/other_midg_gpu.h" ; -+ } -+ @enddot -+ * -+ * -+ * @section sec_base_user_api_gpuprops_kernel Kernel Operation -+ * -+ * During Base Context Create time, user-side makes a single kernel call: -+ * - A call to fill user memory with GPU information structures -+ * -+ * The kernel-side will fill the provided the entire processed @ref base_gpu_props -+ * structure, because this information is required in both -+ * user and kernel side; it does not make sense to decode it twice. -+ * -+ * Coherency groups must be derived from the bitmasks, but this can be done -+ * kernel side, and just once at kernel startup: Coherency groups must already -+ * be known kernel-side, to support chains that specify a 'Only Coherent Group' -+ * SW requirement, or 'Only Coherent Group with Tiler' SW requirement. -+ * -+ * @section sec_base_user_api_gpuprops_cocalc Coherency Group calculation -+ * Creation of the coherent group data is done at device-driver startup, and so -+ * is one-time. This will most likely involve a loop with CLZ, shifting, and -+ * bit clearing on the L2_PRESENT mask, depending on whether the -+ * system is L2 Coherent. The number of shader cores is done by a -+ * population count, since faulty cores may be disabled during production, -+ * producing a non-contiguous mask. -+ * -+ * The memory requirements for this algorithm can be determined either by a u64 -+ * population count on the L2_PRESENT mask (a LUT helper already is -+ * required for the above), or simple assumption that there can be no more than -+ * 16 coherent groups, since core groups are typically 4 cores. -+ */ -+ -+/** -+ * @addtogroup base_user_api_gpuprops User-side Base GPU Property Query APIs -+ * @{ -+ */ -+ -+/** -+ * @addtogroup base_user_api_gpuprops_dyn Dynamic HW Properties -+ * @{ -+ */ -+ -+#define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 3 -+ -+#define BASE_MAX_COHERENT_GROUPS 16 -+ -+struct mali_base_gpu_core_props { -+ /** -+ * Product specific value. -+ */ -+ u32 product_id; -+ -+ /** -+ * Status of the GPU release. -+ * No defined values, but starts at 0 and increases by one for each -+ * release status (alpha, beta, EAC, etc.). -+ * 4 bit values (0-15). -+ */ -+ u16 version_status; -+ -+ /** -+ * Minor release number of the GPU. "P" part of an "RnPn" release number. -+ * 8 bit values (0-255). -+ */ -+ u16 minor_revision; -+ -+ /** -+ * Major release number of the GPU. "R" part of an "RnPn" release number. -+ * 4 bit values (0-15). -+ */ -+ u16 major_revision; -+ -+ u16 padding; -+ -+ /** -+ * This property is deprecated since it has not contained the real current -+ * value of GPU clock speed. It is kept here only for backwards compatibility. -+ * For the new ioctl interface, it is ignored and is treated as a padding -+ * to keep the structure of the same size and retain the placement of its -+ * members. -+ */ -+ u32 gpu_speed_mhz; -+ -+ /** -+ * @usecase GPU clock max/min speed is required for computing best/worst case -+ * in tasks as job scheduling ant irq_throttling. (It is not specified in the -+ * Midgard Architecture). -+ * Also, GPU clock max speed is used for OpenCL's clGetDeviceInfo() function. -+ */ -+ u32 gpu_freq_khz_max; -+ u32 gpu_freq_khz_min; -+ -+ /** -+ * Size of the shader program counter, in bits. -+ */ -+ u32 log2_program_counter_size; -+ -+ /** -+ * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a -+ * bitpattern where a set bit indicates that the format is supported. -+ * -+ * Before using a texture format, it is recommended that the corresponding -+ * bit be checked. -+ */ -+ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; -+ -+ /** -+ * Theoretical maximum memory available to the GPU. It is unlikely that a -+ * client will be able to allocate all of this memory for their own -+ * purposes, but this at least provides an upper bound on the memory -+ * available to the GPU. -+ * -+ * This is required for OpenCL's clGetDeviceInfo() call when -+ * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The -+ * client will not be expecting to allocate anywhere near this value. -+ */ -+ u64 gpu_available_memory_size; -+}; -+ -+/** -+ * -+ * More information is possible - but associativity and bus width are not -+ * required by upper-level apis. -+ */ -+struct mali_base_gpu_l2_cache_props { -+ u8 log2_line_size; -+ u8 log2_cache_size; -+ u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ -+ u8 padding[5]; -+}; -+ -+struct mali_base_gpu_tiler_props { -+ u32 bin_size_bytes; /* Max is 4*2^15 */ -+ u32 max_active_levels; /* Max is 2^15 */ -+}; -+ -+/** -+ * GPU threading system details. -+ */ -+struct mali_base_gpu_thread_props { -+ u32 max_threads; /* Max. number of threads per core */ -+ u32 max_workgroup_size; /* Max. number of threads per workgroup */ -+ u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ -+ u16 max_registers; /* Total size [1..65535] of the register file available per core. */ -+ u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ -+ u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ -+ u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ -+ u8 padding[7]; -+}; -+ -+/** -+ * @brief descriptor for a coherent group -+ * -+ * \c core_mask exposes all cores in that coherent group, and \c num_cores -+ * provides a cached population-count for that mask. -+ * -+ * @note Whilst all cores are exposed in the mask, not all may be available to -+ * the application, depending on the Kernel Power policy. -+ * -+ * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. -+ */ -+struct mali_base_gpu_coherent_group { -+ u64 core_mask; /**< Core restriction mask required for the group */ -+ u16 num_cores; /**< Number of cores in the group */ -+ u16 padding[3]; -+}; -+ -+/** -+ * @brief Coherency group information -+ * -+ * Note that the sizes of the members could be reduced. However, the \c group -+ * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte -+ * aligned, thus leading to wastage if the other members sizes were reduced. -+ * -+ * The groups are sorted by core mask. The core masks are non-repeating and do -+ * not intersect. -+ */ -+struct mali_base_gpu_coherent_group_info { -+ u32 num_groups; -+ -+ /** -+ * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. -+ * -+ * The GPU Counter dumping writes 2048 bytes per core group, regardless of -+ * whether the core groups are coherent or not. Hence this member is needed -+ * to calculate how much memory is required for dumping. -+ * -+ * @note Do not use it to work out how many valid elements are in the -+ * group[] member. Use num_groups instead. -+ */ -+ u32 num_core_groups; -+ -+ /** -+ * Coherency features of the memory, accessed by @ref gpu_mem_features -+ * methods -+ */ -+ u32 coherency; -+ -+ u32 padding; -+ -+ /** -+ * Descriptors of coherent groups -+ */ -+ struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; -+}; -+ -+/** -+ * A complete description of the GPU's Hardware Configuration Discovery -+ * registers. -+ * -+ * The information is presented inefficiently for access. For frequent access, -+ * the values should be better expressed in an unpacked form in the -+ * base_gpu_props structure. -+ * -+ * @usecase The raw properties in @ref gpu_raw_gpu_props are necessary to -+ * allow a user of the Mali Tools (e.g. PAT) to determine "Why is this device -+ * behaving differently?". In this case, all information about the -+ * configuration is potentially useful, but it does not need to be processed -+ * by the driver. Instead, the raw registers can be processed by the Mali -+ * Tools software on the host PC. -+ * -+ */ -+struct gpu_raw_gpu_props { -+ u64 shader_present; -+ u64 tiler_present; -+ u64 l2_present; -+ u64 stack_present; -+ -+ u32 l2_features; -+ u32 suspend_size; /* API 8.2+ */ -+ u32 mem_features; -+ u32 mmu_features; -+ -+ u32 as_present; -+ -+ u32 js_present; -+ u32 js_features[GPU_MAX_JOB_SLOTS]; -+ u32 tiler_features; -+ u32 texture_features[3]; -+ -+ u32 gpu_id; -+ -+ u32 thread_max_threads; -+ u32 thread_max_workgroup_size; -+ u32 thread_max_barrier_size; -+ u32 thread_features; -+ -+ /* -+ * Note: This is the _selected_ coherency mode rather than the -+ * available modes as exposed in the coherency_features register. -+ */ -+ u32 coherency_mode; -+}; -+ -+/** -+ * Return structure for _mali_base_get_gpu_props(). -+ * -+ * NOTE: the raw_props member in this data structure contains the register -+ * values from which the value of the other members are derived. The derived -+ * members exist to allow for efficient access and/or shielding the details -+ * of the layout of the registers. -+ * -+ */ -+typedef struct mali_base_gpu_props { -+ struct mali_base_gpu_core_props core_props; -+ struct mali_base_gpu_l2_cache_props l2_props; -+ u64 unused_1; /* keep for backwards compatibility */ -+ struct mali_base_gpu_tiler_props tiler_props; -+ struct mali_base_gpu_thread_props thread_props; -+ -+ /** This member is large, likely to be 128 bytes */ -+ struct gpu_raw_gpu_props raw_props; -+ -+ /** This must be last member of the structure */ -+ struct mali_base_gpu_coherent_group_info coherency_info; -+} base_gpu_props; -+ -+/** @} end group base_user_api_gpuprops_dyn */ -+ -+/** @} end group base_user_api_gpuprops */ -+ -+/** -+ * @addtogroup base_user_api_core User-side Base core APIs -+ * @{ -+ */ -+ -+/** -+ * \enum base_context_create_flags -+ * -+ * Flags to pass to ::base_context_init. -+ * Flags can be ORed together to enable multiple things. -+ * -+ * These share the same space as BASEP_CONTEXT_FLAG_*, and so must -+ * not collide with them. -+ */ -+enum base_context_create_flags { -+ /** No flags set */ -+ BASE_CONTEXT_CREATE_FLAG_NONE = 0, -+ -+ /** Base context is embedded in a cctx object (flag used for CINSTR software counter macros) */ -+ BASE_CONTEXT_CCTX_EMBEDDED = (1u << 0), -+ -+ /** Base context is a 'System Monitor' context for Hardware counters. -+ * -+ * One important side effect of this is that job submission is disabled. */ -+ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED = (1u << 1) -+}; -+ -+/** -+ * Bitpattern describing the ::base_context_create_flags that can be passed to base_context_init() -+ */ -+#define BASE_CONTEXT_CREATE_ALLOWED_FLAGS \ -+ (((u32)BASE_CONTEXT_CCTX_EMBEDDED) | \ -+ ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED)) -+ -+/** -+ * Bitpattern describing the ::base_context_create_flags that can be passed to the kernel -+ */ -+#define BASE_CONTEXT_CREATE_KERNEL_FLAGS \ -+ ((u32)BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) -+ -+/* -+ * Private flags used on the base context -+ * -+ * These start at bit 31, and run down to zero. -+ * -+ * They share the same space as @ref base_context_create_flags, and so must -+ * not collide with them. -+ */ -+/** Private flag tracking whether job descriptor dumping is disabled */ -+#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((u32)(1 << 31)) -+ -+/** @} end group base_user_api_core */ -+ -+/** @} end group base_user_api */ -+ -+/** -+ * @addtogroup base_plat_config_gpuprops Base Platform Config GPU Properties -+ * @{ -+ * -+ * C Pre-processor macros are exposed here to do with Platform -+ * Config. -+ * -+ * These include: -+ * - GPU Properties that are constant on a particular Midgard Family -+ * Implementation e.g. Maximum samples per pixel on Mali-T600. -+ * - General platform config for the GPU, such as the GPU major and minor -+ * revison. -+ */ -+ -+/** @} end group base_plat_config_gpuprops */ -+ -+/** -+ * @addtogroup base_api Base APIs -+ * @{ -+ */ -+ -+/** -+ * @brief The payload for a replay job. This must be in GPU memory. -+ */ -+typedef struct base_jd_replay_payload { -+ /** -+ * Pointer to the first entry in the base_jd_replay_jc list. These -+ * will be replayed in @b reverse order (so that extra ones can be added -+ * to the head in future soft jobs without affecting this soft job) -+ */ -+ u64 tiler_jc_list; -+ -+ /** -+ * Pointer to the fragment job chain. -+ */ -+ u64 fragment_jc; -+ -+ /** -+ * Pointer to the tiler heap free FBD field to be modified. -+ */ -+ u64 tiler_heap_free; -+ -+ /** -+ * Hierarchy mask for the replayed fragment jobs. May be zero. -+ */ -+ u16 fragment_hierarchy_mask; -+ -+ /** -+ * Hierarchy mask for the replayed tiler jobs. May be zero. -+ */ -+ u16 tiler_hierarchy_mask; -+ -+ /** -+ * Default weight to be used for hierarchy levels not in the original -+ * mask. -+ */ -+ u32 hierarchy_default_weight; -+ -+ /** -+ * Core requirements for the tiler job chain -+ */ -+ base_jd_core_req tiler_core_req; -+ -+ /** -+ * Core requirements for the fragment job chain -+ */ -+ base_jd_core_req fragment_core_req; -+} base_jd_replay_payload; -+ -+#ifdef BASE_LEGACY_UK10_2_SUPPORT -+typedef struct base_jd_replay_payload_uk10_2 { -+ u64 tiler_jc_list; -+ u64 fragment_jc; -+ u64 tiler_heap_free; -+ u16 fragment_hierarchy_mask; -+ u16 tiler_hierarchy_mask; -+ u32 hierarchy_default_weight; -+ u16 tiler_core_req; -+ u16 fragment_core_req; -+ u8 padding[4]; -+} base_jd_replay_payload_uk10_2; -+#endif /* BASE_LEGACY_UK10_2_SUPPORT */ -+ -+/** -+ * @brief An entry in the linked list of job chains to be replayed. This must -+ * be in GPU memory. -+ */ -+typedef struct base_jd_replay_jc { -+ /** -+ * Pointer to next entry in the list. A setting of NULL indicates the -+ * end of the list. -+ */ -+ u64 next; -+ -+ /** -+ * Pointer to the job chain. -+ */ -+ u64 jc; -+ -+} base_jd_replay_jc; -+ -+/* Maximum number of jobs allowed in a fragment chain in the payload of a -+ * replay job */ -+#define BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT 256 -+ -+/** @} end group base_api */ -+ -+typedef struct base_profiling_controls { -+ u32 profiling_controls[FBDUMP_CONTROL_MAX]; -+} base_profiling_controls; -+ -+/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, -+ * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) */ -+#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) -+ -+/* Indicate that job dumping is enabled. This could affect certain timers -+ * to account for the performance impact. */ -+#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) -+ -+#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ -+ BASE_TLSTREAM_JOB_DUMPING_ENABLED) -+ -+#endif /* _BASE_KERNEL_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_base_mem_priv.h b/drivers/gpu/arm/midgard/mali_base_mem_priv.h -new file mode 100755 -index 000000000..4a98a72cc ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_base_mem_priv.h -@@ -0,0 +1,52 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _BASE_MEM_PRIV_H_ -+#define _BASE_MEM_PRIV_H_ -+ -+#define BASE_SYNCSET_OP_MSYNC (1U << 0) -+#define BASE_SYNCSET_OP_CSYNC (1U << 1) -+ -+/* -+ * This structure describe a basic memory coherency operation. -+ * It can either be: -+ * @li a sync from CPU to Memory: -+ * - type = ::BASE_SYNCSET_OP_MSYNC -+ * - mem_handle = a handle to the memory object on which the operation -+ * is taking place -+ * - user_addr = the address of the range to be synced -+ * - size = the amount of data to be synced, in bytes -+ * - offset is ignored. -+ * @li a sync from Memory to CPU: -+ * - type = ::BASE_SYNCSET_OP_CSYNC -+ * - mem_handle = a handle to the memory object on which the operation -+ * is taking place -+ * - user_addr = the address of the range to be synced -+ * - size = the amount of data to be synced, in bytes. -+ * - offset is ignored. -+ */ -+struct basep_syncset { -+ base_mem_handle mem_handle; -+ u64 user_addr; -+ u64 size; -+ u8 type; -+ u8 padding[7]; -+}; -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h b/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h -new file mode 100755 -index 000000000..be454a216 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_base_vendor_specific_func.h -@@ -0,0 +1,24 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010, 2012-2013, 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+#ifndef _BASE_VENDOR_SPEC_FUNC_H_ -+#define _BASE_VENDOR_SPEC_FUNC_H_ -+ -+int kbase_get_vendor_specific_cpu_clock_speed(u32 * const); -+ -+#endif /*_BASE_VENDOR_SPEC_FUNC_H_*/ -diff --git a/drivers/gpu/arm/midgard/mali_kbase.h b/drivers/gpu/arm/midgard/mali_kbase.h -new file mode 100755 -index 000000000..0d9bf23dc ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase.h -@@ -0,0 +1,612 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_H_ -+#define _KBASE_H_ -+ -+#include -+ -+#include -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_base_kernel.h" -+#include -+#include -+ -+/* -+ * Include mali_kbase_defs.h first as this provides types needed by other local -+ * header files. -+ */ -+#include "mali_kbase_defs.h" -+ -+#include "mali_kbase_context.h" -+#include "mali_kbase_strings.h" -+#include "mali_kbase_mem_lowlevel.h" -+#include "mali_kbase_trace_timeline.h" -+#include "mali_kbase_js.h" -+#include "mali_kbase_mem.h" -+#include "mali_kbase_utility.h" -+#include "mali_kbase_gpu_memory_debugfs.h" -+#include "mali_kbase_mem_profile_debugfs.h" -+#include "mali_kbase_debug_job_fault.h" -+#include "mali_kbase_jd_debugfs.h" -+#include "mali_kbase_gpuprops.h" -+#include "mali_kbase_jm.h" -+#include "mali_kbase_vinstr.h" -+ -+#include "ipa/mali_kbase_ipa.h" -+ -+#ifdef CONFIG_GPU_TRACEPOINTS -+#include -+#endif -+/** -+ * @page page_base_kernel_main Kernel-side Base (KBase) APIs -+ */ -+ -+/** -+ * @defgroup base_kbase_api Kernel-side Base (KBase) APIs -+ */ -+ -+struct kbase_device *kbase_device_alloc(void); -+/* -+* note: configuration attributes member of kbdev needs to have -+* been setup before calling kbase_device_init -+*/ -+ -+/* -+* API to acquire device list semaphore and return pointer -+* to the device list head -+*/ -+const struct list_head *kbase_dev_list_get(void); -+/* API to release the device list semaphore */ -+void kbase_dev_list_put(const struct list_head *dev_list); -+ -+int kbase_device_init(struct kbase_device * const kbdev); -+void kbase_device_term(struct kbase_device *kbdev); -+void kbase_device_free(struct kbase_device *kbdev); -+int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); -+ -+/* Needed for gator integration and for reporting vsync information */ -+struct kbase_device *kbase_find_device(int minor); -+void kbase_release_device(struct kbase_device *kbdev); -+ -+void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value); -+ -+struct kbase_context * -+kbase_create_context(struct kbase_device *kbdev, bool is_compat); -+void kbase_destroy_context(struct kbase_context *kctx); -+ -+int kbase_jd_init(struct kbase_context *kctx); -+void kbase_jd_exit(struct kbase_context *kctx); -+ -+/** -+ * kbase_jd_submit - Submit atoms to the job dispatcher -+ * -+ * @kctx: The kbase context to submit to -+ * @user_addr: The address in user space of the struct base_jd_atom_v2 array -+ * @nr_atoms: The number of atoms in the array -+ * @stride: sizeof(struct base_jd_atom_v2) -+ * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) -+ * -+ * Return: 0 on success or error code -+ */ -+int kbase_jd_submit(struct kbase_context *kctx, -+ void __user *user_addr, u32 nr_atoms, u32 stride, -+ bool uk6_atom); -+ -+/** -+ * kbase_jd_done_worker - Handle a job completion -+ * @data: a &struct work_struct -+ * -+ * This function requeues the job from the runpool (if it was soft-stopped or -+ * removed from NEXT registers). -+ * -+ * Removes it from the system if it finished/failed/was cancelled. -+ * -+ * Resolves dependencies to add dependent jobs to the context, potentially -+ * starting them if necessary (which may add more references to the context) -+ * -+ * Releases the reference to the context from the no-longer-running job. -+ * -+ * Handles retrying submission outside of IRQ context if it failed from within -+ * IRQ context. -+ */ -+void kbase_jd_done_worker(struct work_struct *data); -+ -+void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, -+ kbasep_js_atom_done_code done_code); -+void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); -+void kbase_jd_zap_context(struct kbase_context *kctx); -+bool jd_done_nolock(struct kbase_jd_atom *katom, -+ struct list_head *completed_jobs_ctx); -+void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); -+bool jd_submit_atom(struct kbase_context *kctx, -+ const struct base_jd_atom_v2 *user_atom, -+ struct kbase_jd_atom *katom); -+void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); -+ -+void kbase_job_done(struct kbase_device *kbdev, u32 done); -+ -+/** -+ * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms -+ * and soft stop them -+ * @kctx: Pointer to context to check. -+ * @katom: Pointer to priority atom. -+ * -+ * Atoms from @kctx on the same job slot as @katom, which have lower priority -+ * than @katom will be soft stopped and put back in the queue, so that atoms -+ * with higher priority can run. -+ * -+ * The hwaccess_lock must be held when calling this function. -+ */ -+void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom); -+ -+void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, -+ struct kbase_jd_atom *target_katom); -+void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, -+ struct kbase_jd_atom *target_katom, u32 sw_flags); -+void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, -+ struct kbase_jd_atom *target_katom); -+void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, -+ base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); -+void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, -+ struct kbase_jd_atom *target_katom); -+ -+void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); -+int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); -+int kbase_event_pending(struct kbase_context *ctx); -+int kbase_event_init(struct kbase_context *kctx); -+void kbase_event_close(struct kbase_context *kctx); -+void kbase_event_cleanup(struct kbase_context *kctx); -+void kbase_event_wakeup(struct kbase_context *kctx); -+ -+int kbase_process_soft_job(struct kbase_jd_atom *katom); -+int kbase_prepare_soft_job(struct kbase_jd_atom *katom); -+void kbase_finish_soft_job(struct kbase_jd_atom *katom); -+void kbase_cancel_soft_job(struct kbase_jd_atom *katom); -+void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); -+void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); -+#endif -+int kbase_soft_event_update(struct kbase_context *kctx, -+ u64 event, -+ unsigned char new_status); -+ -+bool kbase_replay_process(struct kbase_jd_atom *katom); -+ -+void kbasep_soft_job_timeout_worker(struct timer_list *t); -+void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); -+ -+/* api used internally for register access. Contains validation and tracing */ -+void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value); -+int kbase_device_trace_buffer_install( -+ struct kbase_context *kctx, u32 *tb, size_t size); -+void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx); -+ -+/* api to be ported per OS, only need to do the raw register access */ -+void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value); -+u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset); -+ -+void kbasep_as_do_poke(struct work_struct *work); -+ -+/** Returns the name associated with a Mali exception code -+ * -+ * This function is called from the interrupt handler when a GPU fault occurs. -+ * It reports the details of the fault using KBASE_DEBUG_PRINT_WARN. -+ * -+ * @param[in] kbdev The kbase device that the GPU fault occurred from. -+ * @param[in] exception_code exception code -+ * @return name associated with the exception code -+ */ -+const char *kbase_exception_name(struct kbase_device *kbdev, -+ u32 exception_code); -+ -+/** -+ * Check whether a system suspend is in progress, or has already been suspended -+ * -+ * The caller should ensure that either kbdev->pm.active_count_lock is held, or -+ * a dmb was executed recently (to ensure the value is most -+ * up-to-date). However, without a lock the value could change afterwards. -+ * -+ * @return false if a suspend is not in progress -+ * @return !=false otherwise -+ */ -+static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) -+{ -+ return kbdev->pm.suspending; -+} -+ -+/** -+ * Return the atom's ID, as was originally supplied by userspace in -+ * base_jd_atom_v2::atom_number -+ */ -+static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ int result; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(katom); -+ KBASE_DEBUG_ASSERT(katom->kctx == kctx); -+ -+ result = katom - &kctx->jctx.atoms[0]; -+ KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); -+ return result; -+} -+ -+/** -+ * kbase_jd_atom_from_id - Return the atom structure for the given atom ID -+ * @kctx: Context pointer -+ * @id: ID of atom to retrieve -+ * -+ * Return: Pointer to struct kbase_jd_atom associated with the supplied ID -+ */ -+static inline struct kbase_jd_atom *kbase_jd_atom_from_id( -+ struct kbase_context *kctx, int id) -+{ -+ return &kctx->jctx.atoms[id]; -+} -+ -+/** -+ * Initialize the disjoint state -+ * -+ * The disjoint event count and state are both set to zero. -+ * -+ * Disjoint functions usage: -+ * -+ * The disjoint event count should be incremented whenever a disjoint event occurs. -+ * -+ * There are several cases which are regarded as disjoint behavior. Rather than just increment -+ * the counter during disjoint events we also increment the counter when jobs may be affected -+ * by what the GPU is currently doing. To facilitate this we have the concept of disjoint state. -+ * -+ * Disjoint state is entered during GPU reset and for the entire time that an atom is replaying -+ * (as part of the replay workaround). Increasing the disjoint state also increases the count of -+ * disjoint events. -+ * -+ * The disjoint state is then used to increase the count of disjoint events during job submission -+ * and job completion. Any atom submitted or completed while the disjoint state is greater than -+ * zero is regarded as a disjoint event. -+ * -+ * The disjoint event counter is also incremented immediately whenever a job is soft stopped -+ * and during context creation. -+ * -+ * @param kbdev The kbase device -+ */ -+void kbase_disjoint_init(struct kbase_device *kbdev); -+ -+/** -+ * Increase the count of disjoint events -+ * called when a disjoint event has happened -+ * -+ * @param kbdev The kbase device -+ */ -+void kbase_disjoint_event(struct kbase_device *kbdev); -+ -+/** -+ * Increase the count of disjoint events only if the GPU is in a disjoint state -+ * -+ * This should be called when something happens which could be disjoint if the GPU -+ * is in a disjoint state. The state refcount keeps track of this. -+ * -+ * @param kbdev The kbase device -+ */ -+void kbase_disjoint_event_potential(struct kbase_device *kbdev); -+ -+/** -+ * Returns the count of disjoint events -+ * -+ * @param kbdev The kbase device -+ * @return the count of disjoint events -+ */ -+u32 kbase_disjoint_event_get(struct kbase_device *kbdev); -+ -+/** -+ * Increment the refcount state indicating that the GPU is in a disjoint state. -+ * -+ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) -+ * eventually after the disjoint state has completed @ref kbase_disjoint_state_down -+ * should be called -+ * -+ * @param kbdev The kbase device -+ */ -+void kbase_disjoint_state_up(struct kbase_device *kbdev); -+ -+/** -+ * Decrement the refcount state -+ * -+ * Also Increment the disjoint event count (calls @ref kbase_disjoint_event) -+ * -+ * Called after @ref kbase_disjoint_state_up once the disjoint state is over -+ * -+ * @param kbdev The kbase device -+ */ -+void kbase_disjoint_state_down(struct kbase_device *kbdev); -+ -+/** -+ * If a job is soft stopped and the number of contexts is >= this value -+ * it is reported as a disjoint event -+ */ -+#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 -+ -+#if !defined(UINT64_MAX) -+ #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) -+#endif -+ -+#if KBASE_TRACE_ENABLE -+void kbasep_trace_debugfs_init(struct kbase_device *kbdev); -+ -+#ifndef CONFIG_MALI_SYSTEM_TRACE -+/** Add trace values about a job-slot -+ * -+ * @note Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any -+ * functions called to get the parameters supplied to this macro must: -+ * - be static or static inline -+ * - must just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \ -+ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ -+ KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0) -+ -+/** Add trace values about a job-slot, with info -+ * -+ * @note Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any -+ * functions called to get the parameters supplied to this macro must: -+ * - be static or static inline -+ * - must just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \ -+ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ -+ KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val) -+ -+/** Add trace values about a ctx refcount -+ * -+ * @note Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any -+ * functions called to get the parameters supplied to this macro must: -+ * - be static or static inline -+ * - must just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \ -+ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ -+ KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0) -+/** Add trace values about a ctx refcount, and info -+ * -+ * @note Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any -+ * functions called to get the parameters supplied to this macro must: -+ * - be static or static inline -+ * - must just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \ -+ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ -+ KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val) -+ -+/** Add trace values (no slot or refcount) -+ * -+ * @note Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any -+ * functions called to get the parameters supplied to this macro must: -+ * - be static or static inline -+ * - must just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \ -+ kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \ -+ 0, 0, 0, info_val) -+ -+/** Clear the trace */ -+#define KBASE_TRACE_CLEAR(kbdev) \ -+ kbasep_trace_clear(kbdev) -+ -+/** Dump the slot trace */ -+#define KBASE_TRACE_DUMP(kbdev) \ -+ kbasep_trace_dump(kbdev) -+ -+/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */ -+void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val); -+/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */ -+void kbasep_trace_clear(struct kbase_device *kbdev); -+#else /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ -+/* Dispatch kbase trace events as system trace events */ -+#include -+#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ -+ trace_mali_##code(jobslot, 0) -+ -+#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ -+ trace_mali_##code(jobslot, info_val) -+ -+#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ -+ trace_mali_##code(refcount, 0) -+ -+#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ -+ trace_mali_##code(refcount, info_val) -+ -+#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\ -+ trace_mali_##code(gpu_addr, info_val) -+ -+#define KBASE_TRACE_CLEAR(kbdev)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(0);\ -+ } while (0) -+#define KBASE_TRACE_DUMP(kbdev)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#endif /* #ifndef CONFIG_MALI_SYSTEM_TRACE */ -+#else -+#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(ctx);\ -+ CSTD_UNUSED(katom);\ -+ CSTD_UNUSED(gpu_addr);\ -+ CSTD_UNUSED(jobslot);\ -+ } while (0) -+ -+#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(ctx);\ -+ CSTD_UNUSED(katom);\ -+ CSTD_UNUSED(gpu_addr);\ -+ CSTD_UNUSED(jobslot);\ -+ CSTD_UNUSED(info_val);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(ctx);\ -+ CSTD_UNUSED(katom);\ -+ CSTD_UNUSED(gpu_addr);\ -+ CSTD_UNUSED(refcount);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(ctx);\ -+ CSTD_UNUSED(katom);\ -+ CSTD_UNUSED(gpu_addr);\ -+ CSTD_UNUSED(info_val);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(subcode);\ -+ CSTD_UNUSED(ctx);\ -+ CSTD_UNUSED(katom);\ -+ CSTD_UNUSED(val);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#define KBASE_TRACE_CLEAR(kbdev)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(0);\ -+ } while (0) -+#define KBASE_TRACE_DUMP(kbdev)\ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(0);\ -+ } while (0) -+#endif /* KBASE_TRACE_ENABLE */ -+/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */ -+void kbasep_trace_dump(struct kbase_device *kbdev); -+ -+#ifdef CONFIG_MALI_DEBUG -+/** -+ * kbase_set_driver_inactive - Force driver to go inactive -+ * @kbdev: Device pointer -+ * @inactive: true if driver should go inactive, false otherwise -+ * -+ * Forcing the driver inactive will cause all future IOCTLs to wait until the -+ * driver is made active again. This is intended solely for the use of tests -+ * which require that no jobs are running while the test executes. -+ */ -+void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive); -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ -+#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) -+ -+/* kbase_io_history_init - initialize data struct for register access history -+ * -+ * @kbdev The register history to initialize -+ * @n The number of register accesses that the buffer could hold -+ * -+ * @return 0 if successfully initialized, failure otherwise -+ */ -+int kbase_io_history_init(struct kbase_io_history *h, u16 n); -+ -+/* kbase_io_history_term - uninit all resources for the register access history -+ * -+ * @h The register history to terminate -+ */ -+void kbase_io_history_term(struct kbase_io_history *h); -+ -+/* kbase_io_history_dump - print the register history to the kernel ring buffer -+ * -+ * @kbdev Pointer to kbase_device containing the register history to dump -+ */ -+void kbase_io_history_dump(struct kbase_device *kbdev); -+ -+/** -+ * kbase_io_history_resize - resize the register access history buffer. -+ * -+ * @h: Pointer to a valid register history to resize -+ * @new_size: Number of accesses the buffer could hold -+ * -+ * A successful resize will clear all recent register accesses. -+ * If resizing fails for any reason (e.g., could not allocate memory, invalid -+ * buffer size) then the original buffer will be kept intact. -+ * -+ * @return 0 if the buffer was resized, failure otherwise -+ */ -+int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); -+ -+#else /* CONFIG_DEBUG_FS */ -+ -+#define kbase_io_history_init(...) ((int)0) -+ -+#define kbase_io_history_term CSTD_NOP -+ -+#define kbase_io_history_dump CSTD_NOP -+ -+#define kbase_io_history_resize CSTD_NOP -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+ -+#endif -+ -+ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c -new file mode 100755 -index 000000000..fde0f8ff8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.c -@@ -0,0 +1,209 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2015, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+#include -+#include -+#include -+ -+/* This function is used to solve an HW issue with single iterator GPUs. -+ * If a fragment job is soft-stopped on the edge of its bounding box, can happen that the -+ * restart index is out of bounds and the rerun causes a tile range fault. If this happens -+ * we try to clamp the restart index to a correct value and rerun the job. -+ */ -+/* Mask of X and Y coordinates for the coordinates words in the descriptors*/ -+#define X_COORDINATE_MASK 0x00000FFF -+#define Y_COORDINATE_MASK 0x0FFF0000 -+/* Max number of words needed from the fragment shader job descriptor */ -+#define JOB_HEADER_SIZE_IN_WORDS 10 -+#define JOB_HEADER_SIZE (JOB_HEADER_SIZE_IN_WORDS*sizeof(u32)) -+ -+/* Word 0: Status Word */ -+#define JOB_DESC_STATUS_WORD 0 -+/* Word 1: Restart Index */ -+#define JOB_DESC_RESTART_INDEX_WORD 1 -+/* Word 2: Fault address low word */ -+#define JOB_DESC_FAULT_ADDR_LOW_WORD 2 -+/* Word 8: Minimum Tile Coordinates */ -+#define FRAG_JOB_DESC_MIN_TILE_COORD_WORD 8 -+/* Word 9: Maximum Tile Coordinates */ -+#define FRAG_JOB_DESC_MAX_TILE_COORD_WORD 9 -+ -+int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom) -+{ -+ struct device *dev = katom->kctx->kbdev->dev; -+ u32 clamped = 0; -+ struct kbase_va_region *region; -+ phys_addr_t *page_array; -+ u64 page_index; -+ u32 offset = katom->jc & (~PAGE_MASK); -+ u32 *page_1 = NULL; -+ u32 *page_2 = NULL; -+ u32 job_header[JOB_HEADER_SIZE_IN_WORDS]; -+ void *dst = job_header; -+ u32 minX, minY, maxX, maxY; -+ u32 restartX, restartY; -+ struct page *p; -+ u32 copy_size; -+ -+ dev_warn(dev, "Called TILE_RANGE_FAULT workaround clamping function.\n"); -+ if (!(katom->core_req & BASE_JD_REQ_FS)) -+ return 0; -+ -+ kbase_gpu_vm_lock(katom->kctx); -+ region = kbase_region_tracker_find_region_enclosing_address(katom->kctx, -+ katom->jc); -+ if (!region || (region->flags & KBASE_REG_FREE)) -+ goto out_unlock; -+ -+ page_array = kbase_get_cpu_phy_pages(region); -+ if (!page_array) -+ goto out_unlock; -+ -+ page_index = (katom->jc >> PAGE_SHIFT) - region->start_pfn; -+ -+ p = pfn_to_page(PFN_DOWN(page_array[page_index])); -+ -+ /* we need the first 10 words of the fragment shader job descriptor. -+ * We need to check that the offset + 10 words is less that the page -+ * size otherwise we need to load the next page. -+ * page_size_overflow will be equal to 0 in case the whole descriptor -+ * is within the page > 0 otherwise. -+ */ -+ copy_size = MIN(PAGE_SIZE - offset, JOB_HEADER_SIZE); -+ -+ page_1 = kmap_atomic(p); -+ -+ /* page_1 is a u32 pointer, offset is expressed in bytes */ -+ page_1 += offset>>2; -+ -+ kbase_sync_single_for_cpu(katom->kctx->kbdev, -+ kbase_dma_addr(p) + offset, -+ copy_size, DMA_BIDIRECTIONAL); -+ -+ memcpy(dst, page_1, copy_size); -+ -+ /* The data needed overflows page the dimension, -+ * need to map the subsequent page */ -+ if (copy_size < JOB_HEADER_SIZE) { -+ p = pfn_to_page(PFN_DOWN(page_array[page_index + 1])); -+ page_2 = kmap_atomic(p); -+ -+ kbase_sync_single_for_cpu(katom->kctx->kbdev, -+ kbase_dma_addr(p), -+ JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL); -+ -+ memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size); -+ } -+ -+ /* We managed to correctly map one or two pages (in case of overflow) */ -+ /* Get Bounding Box data and restart index from fault address low word */ -+ minX = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & X_COORDINATE_MASK; -+ minY = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & Y_COORDINATE_MASK; -+ maxX = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & X_COORDINATE_MASK; -+ maxY = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & Y_COORDINATE_MASK; -+ restartX = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & X_COORDINATE_MASK; -+ restartY = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & Y_COORDINATE_MASK; -+ -+ dev_warn(dev, "Before Clamping:\n" -+ "Jobstatus: %08x\n" -+ "restartIdx: %08x\n" -+ "Fault_addr_low: %08x\n" -+ "minCoordsX: %08x minCoordsY: %08x\n" -+ "maxCoordsX: %08x maxCoordsY: %08x\n", -+ job_header[JOB_DESC_STATUS_WORD], -+ job_header[JOB_DESC_RESTART_INDEX_WORD], -+ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], -+ minX, minY, -+ maxX, maxY); -+ -+ /* Set the restart index to the one which generated the fault*/ -+ job_header[JOB_DESC_RESTART_INDEX_WORD] = -+ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD]; -+ -+ if (restartX < minX) { -+ job_header[JOB_DESC_RESTART_INDEX_WORD] = (minX) | restartY; -+ dev_warn(dev, -+ "Clamping restart X index to minimum. %08x clamped to %08x\n", -+ restartX, minX); -+ clamped = 1; -+ } -+ if (restartY < minY) { -+ job_header[JOB_DESC_RESTART_INDEX_WORD] = (minY) | restartX; -+ dev_warn(dev, -+ "Clamping restart Y index to minimum. %08x clamped to %08x\n", -+ restartY, minY); -+ clamped = 1; -+ } -+ if (restartX > maxX) { -+ job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxX) | restartY; -+ dev_warn(dev, -+ "Clamping restart X index to maximum. %08x clamped to %08x\n", -+ restartX, maxX); -+ clamped = 1; -+ } -+ if (restartY > maxY) { -+ job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxY) | restartX; -+ dev_warn(dev, -+ "Clamping restart Y index to maximum. %08x clamped to %08x\n", -+ restartY, maxY); -+ clamped = 1; -+ } -+ -+ if (clamped) { -+ /* Reset the fault address low word -+ * and set the job status to STOPPED */ -+ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] = 0x0; -+ job_header[JOB_DESC_STATUS_WORD] = BASE_JD_EVENT_STOPPED; -+ dev_warn(dev, "After Clamping:\n" -+ "Jobstatus: %08x\n" -+ "restartIdx: %08x\n" -+ "Fault_addr_low: %08x\n" -+ "minCoordsX: %08x minCoordsY: %08x\n" -+ "maxCoordsX: %08x maxCoordsY: %08x\n", -+ job_header[JOB_DESC_STATUS_WORD], -+ job_header[JOB_DESC_RESTART_INDEX_WORD], -+ job_header[JOB_DESC_FAULT_ADDR_LOW_WORD], -+ minX, minY, -+ maxX, maxY); -+ -+ /* Flush CPU cache to update memory for future GPU reads*/ -+ memcpy(page_1, dst, copy_size); -+ p = pfn_to_page(PFN_DOWN(page_array[page_index])); -+ -+ kbase_sync_single_for_device(katom->kctx->kbdev, -+ kbase_dma_addr(p) + offset, -+ copy_size, DMA_TO_DEVICE); -+ -+ if (copy_size < JOB_HEADER_SIZE) { -+ memcpy(page_2, dst + copy_size, -+ JOB_HEADER_SIZE - copy_size); -+ p = pfn_to_page(PFN_DOWN(page_array[page_index + 1])); -+ -+ kbase_sync_single_for_device(katom->kctx->kbdev, -+ kbase_dma_addr(p), -+ JOB_HEADER_SIZE - copy_size, -+ DMA_TO_DEVICE); -+ } -+ } -+ if (copy_size < JOB_HEADER_SIZE) -+ kunmap_atomic(page_2); -+ -+ kunmap_atomic(page_1); -+ -+out_unlock: -+ kbase_gpu_vm_unlock(katom->kctx); -+ return clamped; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h -new file mode 100755 -index 000000000..099a29861 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_10969_workaround.h -@@ -0,0 +1,23 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_10969_WORKAROUND_ -+#define _KBASE_10969_WORKAROUND_ -+ -+int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom); -+ -+#endif /* _KBASE_10969_WORKAROUND_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c -new file mode 100755 -index 000000000..f910fe970 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c -@@ -0,0 +1,102 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+ -+#include -+#include -+ -+#ifdef CONFIG_DEBUG_FS -+#ifdef CONFIG_MALI_DEBUG -+ -+static int kbase_as_fault_read(struct seq_file *sfile, void *data) -+{ -+ uintptr_t as_no = (uintptr_t) sfile->private; -+ -+ struct list_head *entry; -+ const struct list_head *kbdev_list; -+ struct kbase_device *kbdev = NULL; -+ -+ kbdev_list = kbase_dev_list_get(); -+ -+ list_for_each(entry, kbdev_list) { -+ kbdev = list_entry(entry, struct kbase_device, entry); -+ -+ if(kbdev->debugfs_as_read_bitmap & (1ULL << as_no)) { -+ -+ /* don't show this one again until another fault occors */ -+ kbdev->debugfs_as_read_bitmap &= ~(1ULL << as_no); -+ -+ /* output the last page fault addr */ -+ seq_printf(sfile, "%llu\n", (u64) kbdev->as[as_no].fault_addr); -+ } -+ -+ } -+ -+ kbase_dev_list_put(kbdev_list); -+ -+ return 0; -+} -+ -+static int kbase_as_fault_debugfs_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbase_as_fault_read , in->i_private); -+} -+ -+static const struct file_operations as_fault_fops = { -+ .open = kbase_as_fault_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+#endif /* CONFIG_MALI_DEBUG */ -+#endif /* CONFIG_DEBUG_FS */ -+ -+/* -+ * Initialize debugfs entry for each address space -+ */ -+void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) -+{ -+#ifdef CONFIG_DEBUG_FS -+#ifdef CONFIG_MALI_DEBUG -+ uint i; -+ char as_name[64]; -+ struct dentry *debugfs_directory; -+ -+ kbdev->debugfs_as_read_bitmap = 0ULL; -+ -+ KBASE_DEBUG_ASSERT(kbdev->nr_hw_address_spaces); -+ KBASE_DEBUG_ASSERT(sizeof(kbdev->as[0].fault_addr) == sizeof(u64)); -+ -+ debugfs_directory = debugfs_create_dir("address_spaces", -+ kbdev->mali_debugfs_directory); -+ -+ if(debugfs_directory) { -+ for(i = 0; i < kbdev->nr_hw_address_spaces; i++) { -+ snprintf(as_name, ARRAY_SIZE(as_name), "as%u", i); -+ debugfs_create_file(as_name, S_IRUGO, -+ debugfs_directory, (void*) ((uintptr_t) i), &as_fault_fops); -+ } -+ } -+ else -+ dev_warn(kbdev->dev, "unable to create address_spaces debugfs directory"); -+ -+#endif /* CONFIG_MALI_DEBUG */ -+#endif /* CONFIG_DEBUG_FS */ -+ return; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h -new file mode 100755 -index 000000000..3ed224889 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h -@@ -0,0 +1,45 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_AS_FAULT_DEBUG_FS_H -+#define _KBASE_AS_FAULT_DEBUG_FS_H -+ -+/** -+ * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults -+ * -+ * @kbdev: Pointer to kbase_device -+ */ -+void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_as_fault_debugfs_new() - make the last fault available on debugfs -+ * -+ * @kbdev: Pointer to kbase_device -+ * @as_no: The address space the fault occurred on -+ */ -+static inline void -+kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) -+{ -+#ifdef CONFIG_DEBUG_FS -+#ifdef CONFIG_MALI_DEBUG -+ kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); -+#endif /* CONFIG_DEBUG_FS */ -+#endif /* CONFIG_MALI_DEBUG */ -+ return; -+} -+ -+#endif /*_KBASE_AS_FAULT_DEBUG_FS_H*/ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c -new file mode 100755 -index 000000000..c67b3e97f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c -@@ -0,0 +1,64 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Cache Policy API. -+ */ -+ -+#include "mali_kbase_cache_policy.h" -+ -+/* -+ * The output flags should be a combination of the following values: -+ * KBASE_REG_CPU_CACHED: CPU cache should be enabled. -+ */ -+u32 kbase_cache_enabled(u32 flags, u32 nr_pages) -+{ -+ u32 cache_flags = 0; -+ -+ CSTD_UNUSED(nr_pages); -+ -+ if (flags & BASE_MEM_CACHED_CPU) -+ cache_flags |= KBASE_REG_CPU_CACHED; -+ -+ return cache_flags; -+} -+ -+ -+void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, -+ size_t size, enum dma_data_direction dir) -+{ -+/* Check if kernel is using coherency with GPU */ -+#ifdef CONFIG_MALI_COH_KERN -+ if (kbdev->system_coherency == COHERENCY_ACE) -+ return; -+#endif /* CONFIG_MALI_COH_KERN */ -+ dma_sync_single_for_device(kbdev->dev, handle, size, dir); -+} -+ -+ -+void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, -+ size_t size, enum dma_data_direction dir) -+{ -+/* Check if kernel is using coherency with GPU */ -+#ifdef CONFIG_MALI_COH_KERN -+ if (kbdev->system_coherency == COHERENCY_ACE) -+ return; -+#endif /* CONFIG_MALI_COH_KERN */ -+ dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h -new file mode 100755 -index 000000000..0c18bdb35 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h -@@ -0,0 +1,45 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Cache Policy API. -+ */ -+ -+#ifndef _KBASE_CACHE_POLICY_H_ -+#define _KBASE_CACHE_POLICY_H_ -+ -+#include "mali_kbase.h" -+#include "mali_base_kernel.h" -+ -+/** -+ * kbase_cache_enabled - Choose the cache policy for a specific region -+ * @flags: flags describing attributes of the region -+ * @nr_pages: total number of pages (backed or not) for the region -+ * -+ * Tells whether the CPU and GPU caches should be enabled or not for a specific -+ * region. -+ * This function can be modified to customize the cache policy depending on the -+ * flags and size of the region. -+ * -+ * Return: a combination of %KBASE_REG_CPU_CACHED and %KBASE_REG_GPU_CACHED -+ * depending on the cache policy -+ */ -+u32 kbase_cache_enabled(u32 flags, u32 nr_pages); -+ -+#endif /* _KBASE_CACHE_POLICY_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_config.c b/drivers/gpu/arm/midgard/mali_kbase_config.c -new file mode 100755 -index 000000000..fb615ae02 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_config.c -@@ -0,0 +1,51 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+#include -+#include -+ -+int kbasep_platform_device_init(struct kbase_device *kbdev) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_init_func) -+ return platform_funcs_p->platform_init_func(kbdev); -+ -+ return 0; -+} -+ -+void kbasep_platform_device_term(struct kbase_device *kbdev) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_term_func) -+ platform_funcs_p->platform_term_func(kbdev); -+} -+ -+int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed) -+{ -+ KBASE_DEBUG_ASSERT(NULL != clock_speed); -+ -+ *clock_speed = 100; -+ return 0; -+} -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_config.h b/drivers/gpu/arm/midgard/mali_kbase_config.h -new file mode 100755 -index 000000000..356d52bcd ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_config.h -@@ -0,0 +1,345 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_config.h -+ * Configuration API and Attributes for KBase -+ */ -+ -+#ifndef _KBASE_CONFIG_H_ -+#define _KBASE_CONFIG_H_ -+ -+#include -+ -+#include -+#include -+ -+/** -+ * @addtogroup base_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup base_kbase_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup kbase_config Configuration API and Attributes -+ * @{ -+ */ -+ -+#include -+ -+/* Forward declaration of struct kbase_device */ -+struct kbase_device; -+ -+/** -+ * kbase_platform_funcs_conf - Specifies platform init/term function pointers -+ * -+ * Specifies the functions pointers for platform specific initialization and -+ * termination. By default no functions are required. No additional platform -+ * specific control is necessary. -+ */ -+struct kbase_platform_funcs_conf { -+ /** -+ * platform_init_func - platform specific init function pointer -+ * @kbdev - kbase_device pointer -+ * -+ * Returns 0 on success, negative error code otherwise. -+ * -+ * Function pointer for platform specific initialization or NULL if no -+ * initialization function is required. At the point this the GPU is -+ * not active and its power and clocks are in unknown (platform specific -+ * state) as kbase doesn't yet have control of power and clocks. -+ * -+ * The platform specific private pointer kbase_device::platform_context -+ * can be accessed (and possibly initialized) in here. -+ */ -+ int (*platform_init_func)(struct kbase_device *kbdev); -+ /** -+ * platform_term_func - platform specific termination function pointer -+ * @kbdev - kbase_device pointer -+ * -+ * Function pointer for platform specific termination or NULL if no -+ * termination function is required. At the point this the GPU will be -+ * idle but still powered and clocked. -+ * -+ * The platform specific private pointer kbase_device::platform_context -+ * can be accessed (and possibly terminated) in here. -+ */ -+ void (*platform_term_func)(struct kbase_device *kbdev); -+}; -+ -+/* -+ * @brief Specifies the callbacks for power management -+ * -+ * By default no callbacks will be made and the GPU must not be powered off. -+ */ -+struct kbase_pm_callback_conf { -+ /** Callback for when the GPU is idle and the power to it can be switched off. -+ * -+ * The system integrator can decide whether to either do nothing, just switch off -+ * the clocks to the GPU, or to completely power down the GPU. -+ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the -+ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). -+ */ -+ void (*power_off_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for when the GPU is about to become active and power must be supplied. -+ * -+ * This function must not return until the GPU is powered and clocked sufficiently for register access to -+ * succeed. The return value specifies whether the GPU was powered down since the call to power_off_callback. -+ * If the GPU state has been lost then this function must return 1, otherwise it should return 0. -+ * The platform specific private pointer kbase_device::platform_context can be accessed and modified in here. It is the -+ * platform \em callbacks responsibility to initialize and terminate this pointer if used (see @ref kbase_platform_funcs_conf). -+ * -+ * The return value of the first call to this function is ignored. -+ * -+ * @return 1 if the GPU state may have been lost, 0 otherwise. -+ */ -+ int (*power_on_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for when the system is requesting a suspend and GPU power -+ * must be switched off. -+ * -+ * Note that if this callback is present, then this may be called -+ * without a preceding call to power_off_callback. Therefore this -+ * callback must be able to take any action that might otherwise happen -+ * in power_off_callback. -+ * -+ * The platform specific private pointer kbase_device::platform_context -+ * can be accessed and modified in here. It is the platform \em -+ * callbacks responsibility to initialize and terminate this pointer if -+ * used (see @ref kbase_platform_funcs_conf). -+ */ -+ void (*power_suspend_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for when the system is resuming from a suspend and GPU -+ * power must be switched on. -+ * -+ * Note that if this callback is present, then this may be called -+ * without a following call to power_on_callback. Therefore this -+ * callback must be able to take any action that might otherwise happen -+ * in power_on_callback. -+ * -+ * The platform specific private pointer kbase_device::platform_context -+ * can be accessed and modified in here. It is the platform \em -+ * callbacks responsibility to initialize and terminate this pointer if -+ * used (see @ref kbase_platform_funcs_conf). -+ */ -+ void (*power_resume_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for handling runtime power management initialization. -+ * -+ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback -+ * will become active from calls made to the OS from within this function. -+ * The runtime calls can be triggered by calls from @ref power_off_callback and @ref power_on_callback. -+ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. -+ * -+ * @return 0 on success, else int error code. -+ */ -+ int (*power_runtime_init_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for handling runtime power management termination. -+ * -+ * The runtime power management callbacks @ref power_runtime_off_callback and @ref power_runtime_on_callback -+ * should no longer be called by the OS on completion of this function. -+ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. -+ */ -+ void (*power_runtime_term_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for runtime power-off power management callback -+ * -+ * For linux this callback will be called by the kernel runtime_suspend callback. -+ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. -+ * -+ * @return 0 on success, else OS error code. -+ */ -+ void (*power_runtime_off_callback)(struct kbase_device *kbdev); -+ -+ /** Callback for runtime power-on power management callback -+ * -+ * For linux this callback will be called by the kernel runtime_resume callback. -+ * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. -+ */ -+ int (*power_runtime_on_callback)(struct kbase_device *kbdev); -+ -+ /* -+ * Optional callback for checking if GPU can be suspended when idle -+ * -+ * This callback will be called by the runtime power management core -+ * when the reference count goes to 0 to provide notification that the -+ * GPU now seems idle. -+ * -+ * If this callback finds that the GPU can't be powered off, or handles -+ * suspend by powering off directly or queueing up a power off, a -+ * non-zero value must be returned to prevent the runtime PM core from -+ * also triggering a suspend. -+ * -+ * Returning 0 will cause the runtime PM core to conduct a regular -+ * autosuspend. -+ * -+ * This callback is optional and if not provided regular autosuspend -+ * will be triggered. -+ * -+ * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use -+ * this feature. -+ * -+ * Return 0 if GPU can be suspended, positive value if it can not be -+ * suspeneded by runtime PM, else OS error code -+ */ -+ int (*power_runtime_idle_callback)(struct kbase_device *kbdev); -+}; -+ -+/** -+ * kbase_cpuprops_get_default_clock_speed - default for CPU_SPEED_FUNC -+ * @clock_speed - see kbase_cpu_clk_speed_func for details on the parameters -+ * -+ * Returns 0 on success, negative error code otherwise. -+ * -+ * Default implementation of CPU_SPEED_FUNC. This function sets clock_speed -+ * to 100, so will be an underestimate for any real system. -+ */ -+int kbase_cpuprops_get_default_clock_speed(u32 * const clock_speed); -+ -+/** -+ * kbase_cpu_clk_speed_func - Type of the function pointer for CPU_SPEED_FUNC -+ * @param clock_speed - pointer to store the current CPU clock speed in MHz -+ * -+ * Returns 0 on success, otherwise negative error code. -+ * -+ * This is mainly used to implement OpenCL's clGetDeviceInfo(). -+ */ -+typedef int (*kbase_cpu_clk_speed_func) (u32 *clock_speed); -+ -+/** -+ * kbase_gpu_clk_speed_func - Type of the function pointer for GPU_SPEED_FUNC -+ * @param clock_speed - pointer to store the current GPU clock speed in MHz -+ * -+ * Returns 0 on success, otherwise negative error code. -+ * When an error is returned the caller assumes maximum GPU speed stored in -+ * gpu_freq_khz_max. -+ * -+ * If the system timer is not available then this function is required -+ * for the OpenCL queue profiling to return correct timing information. -+ * -+ */ -+typedef int (*kbase_gpu_clk_speed_func) (u32 *clock_speed); -+ -+#ifdef CONFIG_OF -+struct kbase_platform_config { -+}; -+#else -+ -+/* -+ * @brief Specifies start and end of I/O memory region. -+ */ -+struct kbase_io_memory_region { -+ u64 start; -+ u64 end; -+}; -+ -+/* -+ * @brief Specifies I/O related resources like IRQs and memory region for I/O operations. -+ */ -+struct kbase_io_resources { -+ u32 job_irq_number; -+ u32 mmu_irq_number; -+ u32 gpu_irq_number; -+ struct kbase_io_memory_region io_memory_region; -+}; -+ -+struct kbase_platform_config { -+ const struct kbase_io_resources *io_resources; -+}; -+ -+#endif /* CONFIG_OF */ -+ -+/** -+ * @brief Gets the pointer to platform config. -+ * -+ * @return Pointer to the platform config -+ */ -+struct kbase_platform_config *kbase_get_platform_config(void); -+ -+/** -+ * kbasep_platform_device_init: - Platform specific call to initialize hardware -+ * @kbdev: kbase device pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine can initialize any hardware and context state that -+ * is required for the GPU block to function. -+ * -+ * Return: 0 if no errors have been found in the config. -+ * Negative error code otherwise. -+ */ -+int kbasep_platform_device_init(struct kbase_device *kbdev); -+ -+/** -+ * kbasep_platform_device_term - Platform specific call to terminate hardware -+ * @kbdev: Kbase device pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine can destroy any platform specific context state and -+ * shut down any hardware functionality that are outside of the Power Management -+ * callbacks. -+ * -+ */ -+void kbasep_platform_device_term(struct kbase_device *kbdev); -+ -+ -+/** -+ * kbase_platform_early_init - Early initialisation of the platform code -+ * -+ * This function will be called when the module is loaded to perform any -+ * early initialisation required by the platform code. Such as reading -+ * platform specific device tree entries for the GPU. -+ * -+ * Return: 0 for success, any other fail causes module initialisation to fail -+ */ -+int kbase_platform_early_init(void); -+ -+#ifndef CONFIG_OF -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+/** -+ * kbase_platform_fake_register - Register a platform device for the GPU -+ * -+ * This can be used to register a platform device on systems where device tree -+ * is not enabled and the platform initialisation code in the kernel doesn't -+ * create the GPU device. Where possible device tree should be used instead. -+ * -+ * Return: 0 for success, any other fail causes module initialisation to fail -+ */ -+int kbase_platform_fake_register(void); -+ -+/** -+ * kbase_platform_fake_unregister - Unregister a fake platform device -+ * -+ * Unregister the platform device created with kbase_platform_fake_register() -+ */ -+void kbase_platform_fake_unregister(void); -+#endif -+#endif -+ -+ /** @} *//* end group kbase_config */ -+ /** @} *//* end group base_kbase_api */ -+ /** @} *//* end group base_api */ -+ -+#endif /* _KBASE_CONFIG_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h b/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h -new file mode 100755 -index 000000000..1cf44b350 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h -@@ -0,0 +1,227 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file mali_kbase_config_defaults.h -+ * -+ * Default values for configuration settings -+ * -+ */ -+ -+#ifndef _KBASE_CONFIG_DEFAULTS_H_ -+#define _KBASE_CONFIG_DEFAULTS_H_ -+ -+/* Include mandatory definitions per platform */ -+#include -+ -+/** -+* Boolean indicating whether the driver is configured to be secure at -+* a potential loss of performance. -+* -+* This currently affects only r0p0-15dev0 HW and earlier. -+* -+* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and -+* performance: -+* -+* - When this is set to true, the driver remains fully secure, -+* but potentially loses performance compared with setting this to -+* false. -+* - When set to false, the driver is open to certain security -+* attacks. -+* -+* From r0p0-00rel0 and onwards, there is no security loss by setting -+* this to false, and no performance loss by setting it to -+* true. -+*/ -+#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false -+ -+enum { -+ /** -+ * Use unrestricted Address ID width on the AXI bus. -+ */ -+ KBASE_AID_32 = 0x0, -+ -+ /** -+ * Restrict GPU to a half of maximum Address ID count. -+ * This will reduce performance, but reduce bus load due to GPU. -+ */ -+ KBASE_AID_16 = 0x3, -+ -+ /** -+ * Restrict GPU to a quarter of maximum Address ID count. -+ * This will reduce performance, but reduce bus load due to GPU. -+ */ -+ KBASE_AID_8 = 0x2, -+ -+ /** -+ * Restrict GPU to an eighth of maximum Address ID count. -+ * This will reduce performance, but reduce bus load due to GPU. -+ */ -+ KBASE_AID_4 = 0x1 -+}; -+ -+/** -+ * Default setting for read Address ID limiting on AXI bus. -+ * -+ * Attached value: u32 register value -+ * KBASE_AID_32 - use the full 32 IDs (5 ID bits) -+ * KBASE_AID_16 - use 16 IDs (4 ID bits) -+ * KBASE_AID_8 - use 8 IDs (3 ID bits) -+ * KBASE_AID_4 - use 4 IDs (2 ID bits) -+ * Default value: KBASE_AID_32 (no limit). Note hardware implementation -+ * may limit to a lower value. -+ */ -+#define DEFAULT_ARID_LIMIT KBASE_AID_32 -+ -+/** -+ * Default setting for write Address ID limiting on AXI. -+ * -+ * Attached value: u32 register value -+ * KBASE_AID_32 - use the full 32 IDs (5 ID bits) -+ * KBASE_AID_16 - use 16 IDs (4 ID bits) -+ * KBASE_AID_8 - use 8 IDs (3 ID bits) -+ * KBASE_AID_4 - use 4 IDs (2 ID bits) -+ * Default value: KBASE_AID_32 (no limit). Note hardware implementation -+ * may limit to a lower value. -+ */ -+#define DEFAULT_AWID_LIMIT KBASE_AID_32 -+ -+/** -+ * Default UMP device mapping. A UMP_DEVICE__SHIFT value which -+ * defines which UMP device this GPU should be mapped to. -+ */ -+#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT -+ -+/* -+ * Default period for DVFS sampling -+ */ -+// #define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ -+#define DEFAULT_PM_DVFS_PERIOD 20 /* 20 ms */ -+ -+/* -+ * Power Management poweroff tick granuality. This is in nanoseconds to -+ * allow HR timer support. -+ * -+ * On each scheduling tick, the power manager core may decide to: -+ * -# Power off one or more shader cores -+ * -# Power off the entire GPU -+ */ -+#define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ -+ -+/* -+ * Power Manager number of ticks before shader cores are powered off -+ */ -+#define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ -+ -+/* -+ * Power Manager number of ticks before GPU is powered off -+ */ -+#define DEFAULT_PM_POWEROFF_TICK_GPU (2) /* 400-800us */ -+ -+/* -+ * Default scheduling tick granuality -+ */ -+#define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ -+ -+/* -+ * Default minimum number of scheduling ticks before jobs are soft-stopped. -+ * -+ * This defines the time-slice for a job (which may be different from that of a -+ * context) -+ */ -+#define DEFAULT_JS_SOFT_STOP_TICKS (1) /* 100ms-200ms */ -+ -+/* -+ * Default minimum number of scheduling ticks before CL jobs are soft-stopped. -+ */ -+#define DEFAULT_JS_SOFT_STOP_TICKS_CL (1) /* 100ms-200ms */ -+ -+/* -+ * Default minimum number of scheduling ticks before jobs are hard-stopped -+ */ -+#define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ -+#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */ -+ -+/* -+ * Default minimum number of scheduling ticks before CL jobs are hard-stopped. -+ */ -+#define DEFAULT_JS_HARD_STOP_TICKS_CL (50) /* 5s */ -+ -+/* -+ * Default minimum number of scheduling ticks before jobs are hard-stopped -+ * during dumping -+ */ -+#define DEFAULT_JS_HARD_STOP_TICKS_DUMPING (15000) /* 1500s */ -+ -+/* -+ * Default timeout for some software jobs, after which the software event wait -+ * jobs will be cancelled. -+ */ -+#define DEFAULT_JS_SOFT_JOB_TIMEOUT (3000) /* 3s */ -+ -+/* -+ * Default minimum number of scheduling ticks before the GPU is reset to clear a -+ * "stuck" job -+ */ -+#define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ -+#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */ -+ -+/* -+ * Default minimum number of scheduling ticks before the GPU is reset to clear a -+ * "stuck" CL job. -+ */ -+#define DEFAULT_JS_RESET_TICKS_CL (55) /* 5.5s */ -+ -+/* -+ * Default minimum number of scheduling ticks before the GPU is reset to clear a -+ * "stuck" job during dumping. -+ */ -+#define DEFAULT_JS_RESET_TICKS_DUMPING (15020) /* 1502s */ -+ -+/* -+ * Default number of milliseconds given for other jobs on the GPU to be -+ * soft-stopped when the GPU needs to be reset. -+ */ -+#define DEFAULT_RESET_TIMEOUT_MS (3000) /* 3s */ -+ -+/* -+ * Default timeslice that a context is scheduled in for, in nanoseconds. -+ * -+ * When a context has used up this amount of time across its jobs, it is -+ * scheduled out to let another run. -+ * -+ * @note the resolution is nanoseconds (ns) here, because that's the format -+ * often used by the OS. -+ */ -+#define DEFAULT_JS_CTX_TIMESLICE_NS (50000000) /* 50ms */ -+ -+/* -+ * Perform GPU power down using only platform specific code, skipping DDK power -+ * management. -+ * -+ * If this is non-zero then kbase will avoid powering down shader cores, the -+ * tiler, and the L2 cache, instead just powering down the entire GPU through -+ * platform specific code. This may be required for certain platform -+ * integrations. -+ * -+ * Note that as this prevents kbase from powering down shader cores, this limits -+ * the available power policies to coarse_demand and always_on. -+ */ -+#define PLATFORM_POWER_DOWN_ONLY (1) -+ -+#endif /* _KBASE_CONFIG_DEFAULTS_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_context.c b/drivers/gpu/arm/midgard/mali_kbase_context.c -new file mode 100755 -index 000000000..6338a7e22 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_context.c -@@ -0,0 +1,342 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Base kernel context APIs -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+/** -+ * kbase_create_context() - Create a kernel base context. -+ * @kbdev: Kbase device -+ * @is_compat: Force creation of a 32-bit context -+ * -+ * Allocate and init a kernel base context. -+ * -+ * Return: new kbase context -+ */ -+struct kbase_context * -+kbase_create_context(struct kbase_device *kbdev, bool is_compat) -+{ -+ struct kbase_context *kctx; -+ int err; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ /* zero-inited as lot of code assume it's zero'ed out on create */ -+ kctx = vzalloc(sizeof(*kctx)); -+ -+ if (!kctx) -+ goto out; -+ -+ /* creating a context is considered a disjoint event */ -+ kbase_disjoint_event(kbdev); -+ -+ kctx->kbdev = kbdev; -+ kctx->as_nr = KBASEP_AS_NR_INVALID; -+ atomic_set(&kctx->refcount, 0); -+ if (is_compat) -+ kbase_ctx_flag_set(kctx, KCTX_COMPAT); -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ kctx->timeline.owner_tgid = task_tgid_nr(current); -+#endif -+ atomic_set(&kctx->setup_complete, 0); -+ atomic_set(&kctx->setup_in_progress, 0); -+ spin_lock_init(&kctx->mm_update_lock); -+ kctx->process_mm = NULL; -+ atomic_set(&kctx->nonmapped_pages, 0); -+ kctx->slots_pullable = 0; -+ kctx->tgid = current->tgid; -+ kctx->pid = current->pid; -+ -+ err = kbase_mem_pool_init(&kctx->mem_pool, -+ kbdev->mem_pool_max_size_default, -+ kctx->kbdev, &kbdev->mem_pool); -+ if (err) -+ goto free_kctx; -+ -+ err = kbase_mem_evictable_init(kctx); -+ if (err) -+ goto free_pool; -+ -+ atomic_set(&kctx->used_pages, 0); -+ -+ err = kbase_jd_init(kctx); -+ if (err) -+ goto deinit_evictable; -+ -+ err = kbasep_js_kctx_init(kctx); -+ if (err) -+ goto free_jd; /* safe to call kbasep_js_kctx_term in this case */ -+ -+ err = kbase_event_init(kctx); -+ if (err) -+ goto free_jd; -+ -+ atomic_set(&kctx->drain_pending, 0); -+ -+ mutex_init(&kctx->reg_lock); -+ -+ INIT_LIST_HEAD(&kctx->waiting_soft_jobs); -+ spin_lock_init(&kctx->waiting_soft_jobs_lock); -+#ifdef CONFIG_KDS -+ INIT_LIST_HEAD(&kctx->waiting_kds_resource); -+#endif -+ err = kbase_dma_fence_init(kctx); -+ if (err) -+ goto free_event; -+ -+ err = kbase_mmu_init(kctx); -+ if (err) -+ goto term_dma_fence; -+ -+ do { -+ err = kbase_mem_pool_grow(&kctx->mem_pool, -+ MIDGARD_MMU_BOTTOMLEVEL); -+ if (err) -+ goto pgd_no_mem; -+ -+ mutex_lock(&kctx->mmu_lock); -+ kctx->pgd = kbase_mmu_alloc_pgd(kctx); -+ mutex_unlock(&kctx->mmu_lock); -+ } while (!kctx->pgd); -+ -+ kctx->aliasing_sink_page = kbase_mem_alloc_page(kctx->kbdev); -+ if (!kctx->aliasing_sink_page) -+ goto no_sink_page; -+ -+ init_waitqueue_head(&kctx->event_queue); -+ -+ kctx->cookies = KBASE_COOKIE_MASK; -+ -+ /* Make sure page 0 is not used... */ -+ err = kbase_region_tracker_init(kctx); -+ if (err) -+ goto no_region_tracker; -+ -+ err = kbase_sticky_resource_init(kctx); -+ if (err) -+ goto no_sticky; -+ -+ err = kbase_jit_init(kctx); -+ if (err) -+ goto no_jit; -+#ifdef CONFIG_GPU_TRACEPOINTS -+ atomic_set(&kctx->jctx.work_id, 0); -+#endif -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ atomic_set(&kctx->timeline.jd_atoms_in_flight, 0); -+#endif -+ -+ kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1; -+ -+ mutex_init(&kctx->vinstr_cli_lock); -+ -+ timer_setup(&kctx->soft_job_timeout, -+ kbasep_soft_job_timeout_worker, -+ 0); -+ -+ return kctx; -+ -+no_jit: -+ kbase_gpu_vm_lock(kctx); -+ kbase_sticky_resource_term(kctx); -+ kbase_gpu_vm_unlock(kctx); -+no_sticky: -+ kbase_region_tracker_term(kctx); -+no_region_tracker: -+ kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false); -+no_sink_page: -+ /* VM lock needed for the call to kbase_mmu_free_pgd */ -+ kbase_gpu_vm_lock(kctx); -+ kbase_mmu_free_pgd(kctx); -+ kbase_gpu_vm_unlock(kctx); -+pgd_no_mem: -+ kbase_mmu_term(kctx); -+term_dma_fence: -+ kbase_dma_fence_term(kctx); -+free_event: -+ kbase_event_cleanup(kctx); -+free_jd: -+ /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ -+ kbasep_js_kctx_term(kctx); -+ kbase_jd_exit(kctx); -+deinit_evictable: -+ kbase_mem_evictable_deinit(kctx); -+free_pool: -+ kbase_mem_pool_term(&kctx->mem_pool); -+free_kctx: -+ vfree(kctx); -+out: -+ return NULL; -+} -+KBASE_EXPORT_SYMBOL(kbase_create_context); -+ -+static void kbase_reg_pending_dtor(struct kbase_va_region *reg) -+{ -+ dev_dbg(reg->kctx->kbdev->dev, "Freeing pending unmapped region\n"); -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+ kfree(reg); -+} -+ -+/** -+ * kbase_destroy_context - Destroy a kernel base context. -+ * @kctx: Context to destroy -+ * -+ * Calls kbase_destroy_os_context() to free OS specific structures. -+ * Will release all outstanding regions. -+ */ -+void kbase_destroy_context(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev; -+ int pages; -+ unsigned long pending_regions_to_clean; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ kbdev = kctx->kbdev; -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ -+ KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u); -+ -+ /* Ensure the core is powered up for the destroy process */ -+ /* A suspend won't happen here, because we're in a syscall from a userspace -+ * thread. */ -+ kbase_pm_context_active(kbdev); -+ -+ kbase_jd_zap_context(kctx); -+ -+#ifdef CONFIG_DEBUG_FS -+ /* Removing the rest of the debugfs entries here as we want to keep the -+ * atom debugfs interface alive until all atoms have completed. This -+ * is useful for debugging hung contexts. */ -+ debugfs_remove_recursive(kctx->kctx_dentry); -+#endif -+ -+ kbase_event_cleanup(kctx); -+ -+ /* -+ * JIT must be terminated before the code below as it must be called -+ * without the region lock being held. -+ * The code above ensures no new JIT allocations can be made by -+ * by the time we get to this point of context tear down. -+ */ -+ kbase_jit_term(kctx); -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ kbase_sticky_resource_term(kctx); -+ -+ /* MMU is disabled as part of scheduling out the context */ -+ kbase_mmu_free_pgd(kctx); -+ -+ /* drop the aliasing sink page now that it can't be mapped anymore */ -+ kbase_mem_pool_free(&kctx->mem_pool, kctx->aliasing_sink_page, false); -+ -+ /* free pending region setups */ -+ pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK; -+ while (pending_regions_to_clean) { -+ unsigned int cookie = __ffs(pending_regions_to_clean); -+ -+ BUG_ON(!kctx->pending_regions[cookie]); -+ -+ kbase_reg_pending_dtor(kctx->pending_regions[cookie]); -+ -+ kctx->pending_regions[cookie] = NULL; -+ pending_regions_to_clean &= ~(1UL << cookie); -+ } -+ -+ kbase_region_tracker_term(kctx); -+ kbase_gpu_vm_unlock(kctx); -+ -+ /* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */ -+ kbasep_js_kctx_term(kctx); -+ -+ kbase_jd_exit(kctx); -+ -+ kbase_pm_context_idle(kbdev); -+ -+ kbase_dma_fence_term(kctx); -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); -+ kbase_ctx_sched_remove_ctx(kctx); -+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ kbase_mmu_term(kctx); -+ -+ pages = atomic_read(&kctx->used_pages); -+ if (pages != 0) -+ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); -+ -+ kbase_mem_evictable_deinit(kctx); -+ kbase_mem_pool_term(&kctx->mem_pool); -+ WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); -+ -+ vfree(kctx); -+} -+KBASE_EXPORT_SYMBOL(kbase_destroy_context); -+ -+/** -+ * kbase_context_set_create_flags - Set creation flags on a context -+ * @kctx: Kbase context -+ * @flags: Flags to set -+ * -+ * Return: 0 on success -+ */ -+int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags) -+{ -+ int err = 0; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ unsigned long irq_flags; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ /* Validate flags */ -+ if (flags != (flags & BASE_CONTEXT_CREATE_KERNEL_FLAGS)) { -+ err = -EINVAL; -+ goto out; -+ } -+ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); -+ -+ /* Translate the flags */ -+ if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) -+ kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); -+ -+ /* Latch the initial attributes into the Job Scheduler */ -+ kbasep_js_ctx_attr_set_initial_attrs(kctx->kbdev, kctx); -+ -+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ out: -+ return err; -+} -+KBASE_EXPORT_SYMBOL(kbase_context_set_create_flags); -diff --git a/drivers/gpu/arm/midgard/mali_kbase_context.h b/drivers/gpu/arm/midgard/mali_kbase_context.h -new file mode 100755 -index 000000000..a3f5bb0ce ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_context.h -@@ -0,0 +1,90 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_CONTEXT_H_ -+#define _KBASE_CONTEXT_H_ -+ -+#include -+ -+ -+int kbase_context_set_create_flags(struct kbase_context *kctx, u32 flags); -+ -+/** -+ * kbase_ctx_flag - Check if @flag is set on @kctx -+ * @kctx: Pointer to kbase context to check -+ * @flag: Flag to check -+ * -+ * Return: true if @flag is set on @kctx, false if not. -+ */ -+static inline bool kbase_ctx_flag(struct kbase_context *kctx, -+ enum kbase_context_flags flag) -+{ -+ return atomic_read(&kctx->flags) & flag; -+} -+ -+/** -+ * kbase_ctx_flag_clear - Clear @flag on @kctx -+ * @kctx: Pointer to kbase context -+ * @flag: Flag to clear -+ * -+ * Clear the @flag on @kctx. This is done atomically, so other flags being -+ * cleared or set at the same time will be safe. -+ * -+ * Some flags have locking requirements, check the documentation for the -+ * respective flags. -+ */ -+static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, -+ enum kbase_context_flags flag) -+{ -+#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE -+ /* -+ * Earlier kernel versions doesn't have atomic_andnot() or -+ * atomic_and(). atomic_clear_mask() was only available on some -+ * architectures and removed on arm in v3.13 on arm and arm64. -+ * -+ * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, -+ * when atomic_andnot() becomes available. -+ */ -+ int old, new; -+ -+ do { -+ old = atomic_read(&kctx->flags); -+ new = old & ~flag; -+ -+ } while (atomic_cmpxchg(&kctx->flags, old, new) != old); -+#else -+ atomic_andnot(flag, &kctx->flags); -+#endif -+} -+ -+/** -+ * kbase_ctx_flag_set - Set @flag on @kctx -+ * @kctx: Pointer to kbase context -+ * @flag: Flag to clear -+ * -+ * Set the @flag on @kctx. This is done atomically, so other flags being -+ * cleared or set at the same time will be safe. -+ * -+ * Some flags have locking requirements, check the documentation for the -+ * respective flags. -+ */ -+static inline void kbase_ctx_flag_set(struct kbase_context *kctx, -+ enum kbase_context_flags flag) -+{ -+ atomic_or(flag, &kctx->flags); -+} -+#endif /* _KBASE_CONTEXT_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_core_linux.c b/drivers/gpu/arm/midgard/mali_kbase_core_linux.c -new file mode 100755 -index 000000000..da55cb080 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_core_linux.c -@@ -0,0 +1,4990 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+#define ENABLE_DEBUG_LOG -+#include "platform/rk/custom_log.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#ifdef CONFIG_MALI_DEVFREQ -+#include -+#include -+#ifdef CONFIG_DEVFREQ_THERMAL -+#include -+#endif /* CONFIG_DEVFREQ_THERMAL */ -+#endif /* CONFIG_MALI_DEVFREQ */ -+#ifdef CONFIG_MALI_NO_MALI -+#include "mali_kbase_model_linux.h" -+#endif /* CONFIG_MALI_NO_MALI */ -+#include "mali_kbase_mem_profile_debugfs_buf_size.h" -+#include "mali_kbase_debug_mem_view.h" -+#include "mali_kbase_mem.h" -+#include "mali_kbase_mem_pool_debugfs.h" -+#if !MALI_CUSTOMER_RELEASE -+#include "mali_kbase_regs_dump_debugfs.h" -+#endif /* !MALI_CUSTOMER_RELEASE */ -+#include "mali_kbase_regs_history_debugfs.h" -+#include -+#include -+#include -+#include -+#include "mali_kbase_ioctl.h" -+ -+#ifdef CONFIG_KDS -+#include -+#include -+#include -+#endif /* CONFIG_KDS */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include /* is_compat_task */ -+#include -+#include -+#ifdef CONFIG_MALI_PLATFORM_DEVICETREE -+#include -+#endif /* CONFIG_MALI_PLATFORM_DEVICETREE */ -+#include -+#include -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+#include -+#endif /*CONFIG_MALI_PLATFORM_FAKE */ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#include -+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+#include -+#include -+ -+#include -+ -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) -+#include -+#else -+#include -+#endif -+ -+#include -+ -+#include -+ -+/* GPU IRQ Tags */ -+#define JOB_IRQ_TAG 0 -+#define MMU_IRQ_TAG 1 -+#define GPU_IRQ_TAG 2 -+ -+#if MALI_UNIT_TEST -+static struct kbase_exported_test_data shared_kernel_test_data; -+EXPORT_SYMBOL(shared_kernel_test_data); -+#endif /* MALI_UNIT_TEST */ -+ -+/** rk_ext : version of rk_ext on mali_ko, aka. rk_ko_ver. */ -+#define ROCKCHIP_VERSION (13) -+ -+static int kbase_dev_nr; -+ -+static DEFINE_MUTEX(kbase_dev_list_lock); -+static LIST_HEAD(kbase_dev_list); -+ -+#define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" -+static inline void __compile_time_asserts(void) -+{ -+ CSTD_COMPILE_TIME_ASSERT(sizeof(KERNEL_SIDE_DDK_VERSION_STRING) <= KBASE_GET_VERSION_BUFFER_SIZE); -+} -+ -+static int kbase_api_handshake(struct kbase_context *kctx, -+ struct kbase_ioctl_version_check *version) -+{ -+ switch (version->major) { -+#ifdef BASE_LEGACY_UK6_SUPPORT -+ case 6: -+ /* We are backwards compatible with version 6, -+ * so pretend to be the old version */ -+ version->major = 6; -+ version->minor = 1; -+ break; -+#endif /* BASE_LEGACY_UK6_SUPPORT */ -+#ifdef BASE_LEGACY_UK7_SUPPORT -+ case 7: -+ /* We are backwards compatible with version 7, -+ * so pretend to be the old version */ -+ version->major = 7; -+ version->minor = 1; -+ break; -+#endif /* BASE_LEGACY_UK7_SUPPORT */ -+#ifdef BASE_LEGACY_UK8_SUPPORT -+ case 8: -+ /* We are backwards compatible with version 8, -+ * so pretend to be the old version */ -+ version->major = 8; -+ version->minor = 4; -+ break; -+#endif /* BASE_LEGACY_UK8_SUPPORT */ -+#ifdef BASE_LEGACY_UK9_SUPPORT -+ case 9: -+ /* We are backwards compatible with version 9, -+ * so pretend to be the old version */ -+ version->major = 9; -+ version->minor = 0; -+ break; -+#endif /* BASE_LEGACY_UK8_SUPPORT */ -+ case BASE_UK_VERSION_MAJOR: -+ /* set minor to be the lowest common */ -+ version->minor = min_t(int, BASE_UK_VERSION_MINOR, -+ (int)version->minor); -+ break; -+ default: -+ /* We return our actual version regardless if it -+ * matches the version returned by userspace - -+ * userspace can bail if it can't handle this -+ * version */ -+ version->major = BASE_UK_VERSION_MAJOR; -+ version->minor = BASE_UK_VERSION_MINOR; -+ break; -+ } -+ -+ /* save the proposed version number for later use */ -+ kctx->api_version = KBASE_API_VERSION(version->major, version->minor); -+ -+ return 0; -+} -+ -+/** -+ * enum mali_error - Mali error codes shared with userspace -+ * -+ * This is subset of those common Mali errors that can be returned to userspace. -+ * Values of matching user and kernel space enumerators MUST be the same. -+ * MALI_ERROR_NONE is guaranteed to be 0. -+ * -+ * @MALI_ERROR_NONE: Success -+ * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver -+ * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure -+ * @MALI_ERROR_FUNCTION_FAILED: Generic error code -+ */ -+enum mali_error { -+ MALI_ERROR_NONE = 0, -+ MALI_ERROR_OUT_OF_GPU_MEMORY, -+ MALI_ERROR_OUT_OF_MEMORY, -+ MALI_ERROR_FUNCTION_FAILED, -+}; -+ -+enum { -+ inited_mem = (1u << 0), -+ inited_js = (1u << 1), -+ inited_pm_runtime_init = (1u << 2), -+#ifdef CONFIG_MALI_DEVFREQ -+ inited_devfreq = (1u << 3), -+#endif /* CONFIG_MALI_DEVFREQ */ -+ inited_tlstream = (1u << 4), -+ inited_backend_early = (1u << 5), -+ inited_backend_late = (1u << 6), -+ inited_device = (1u << 7), -+ inited_vinstr = (1u << 8), -+ -+ inited_job_fault = (1u << 10), -+ inited_sysfs_group = (1u << 11), -+ inited_misc_register = (1u << 12), -+ inited_get_device = (1u << 13), -+ inited_dev_list = (1u << 14), -+ inited_debugfs = (1u << 15), -+ inited_gpu_device = (1u << 16), -+ inited_registers_map = (1u << 17), -+ inited_io_history = (1u << 18), -+ inited_power_control = (1u << 19), -+ inited_buslogger = (1u << 20), -+ inited_protected = (1u << 21), -+ inited_ctx_sched = (1u << 22) -+}; -+ -+ -+#ifdef CONFIG_MALI_DEBUG -+#define INACTIVE_WAIT_MS (5000) -+ -+void kbase_set_driver_inactive(struct kbase_device *kbdev, bool inactive) -+{ -+ kbdev->driver_inactive = inactive; -+ wake_up(&kbdev->driver_inactive_wait); -+ -+ /* Wait for any running IOCTLs to complete */ -+ if (inactive) -+ msleep(INACTIVE_WAIT_MS); -+} -+KBASE_EXPORT_TEST_API(kbase_set_driver_inactive); -+#endif /* CONFIG_MALI_DEBUG */ -+ -+/** -+ * kbase_legacy_dispatch - UKK dispatch function -+ * -+ * This is the dispatch function for the legacy UKK ioctl interface. No new -+ * ioctls should be added to this function, see kbase_ioctl instead. -+ * -+ * @kctx: The kernel context structure -+ * @args: Pointer to the data structure passed from/to user space -+ * @args_size: Size of the data structure -+ */ -+static int kbase_legacy_dispatch(struct kbase_context *kctx, -+ void * const args, u32 args_size) -+{ -+ struct kbase_device *kbdev; -+ union uk_header *ukh = args; -+ u32 id; -+ int ret = 0; -+ -+ KBASE_DEBUG_ASSERT(ukh != NULL); -+ -+ kbdev = kctx->kbdev; -+ id = ukh->id; -+ ukh->ret = MALI_ERROR_NONE; /* Be optimistic */ -+ -+#ifdef CONFIG_MALI_DEBUG -+ wait_event(kbdev->driver_inactive_wait, -+ kbdev->driver_inactive == false); -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ if (UKP_FUNC_ID_CHECK_VERSION == id) { -+ struct uku_version_check_args *version_check; -+ struct kbase_ioctl_version_check version; -+ -+ if (args_size != sizeof(struct uku_version_check_args)) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ return 0; -+ } -+ version_check = (struct uku_version_check_args *)args; -+ version.minor = version_check->minor; -+ version.major = version_check->major; -+ -+ kbase_api_handshake(kctx, &version); -+ -+ version_check->minor = version.minor; -+ version_check->major = version.major; -+ ukh->ret = MALI_ERROR_NONE; -+ return 0; -+ } -+ -+ /* block calls until version handshake */ -+ if (kctx->api_version == 0) -+ return -EINVAL; -+ -+ if (!atomic_read(&kctx->setup_complete)) { -+ struct kbase_uk_set_flags *kbase_set_flags; -+ -+ /* setup pending, try to signal that we'll do the setup, -+ * if setup was already in progress, err this call -+ */ -+ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) -+ return -EINVAL; -+ -+ /* if unexpected call, will stay stuck in setup mode -+ * (is it the only call we accept?) -+ */ -+ if (id != KBASE_FUNC_SET_FLAGS) -+ return -EINVAL; -+ -+ kbase_set_flags = (struct kbase_uk_set_flags *)args; -+ -+ /* if not matching the expected call, stay in setup mode */ -+ if (sizeof(*kbase_set_flags) != args_size) -+ goto bad_size; -+ -+ /* if bad flags, will stay stuck in setup mode */ -+ if (kbase_context_set_create_flags(kctx, -+ kbase_set_flags->create_flags) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ -+ atomic_set(&kctx->setup_complete, 1); -+ return 0; -+ } -+ -+ /* setup complete, perform normal operation */ -+ switch (id) { -+ case KBASE_FUNC_MEM_JIT_INIT: -+ { -+ struct kbase_uk_mem_jit_init *jit_init = args; -+ -+ if (sizeof(*jit_init) != args_size) -+ goto bad_size; -+ -+ if (kbase_region_tracker_init_jit(kctx, -+ jit_init->va_pages)) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ case KBASE_FUNC_MEM_ALLOC: -+ { -+ struct kbase_uk_mem_alloc *mem = args; -+ struct kbase_va_region *reg; -+ -+ if (sizeof(*mem) != args_size) -+ goto bad_size; -+ -+#if defined(CONFIG_64BIT) -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ /* force SAME_VA if a 64-bit client */ -+ mem->flags |= BASE_MEM_SAME_VA; -+ } -+#endif -+ -+ reg = kbase_mem_alloc(kctx, mem->va_pages, -+ mem->commit_pages, mem->extent, -+ &mem->flags, &mem->gpu_va); -+ mem->va_alignment = 0; -+ -+ if (!reg) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ case KBASE_FUNC_MEM_IMPORT: { -+ struct kbase_uk_mem_import *mem_import = args; -+ void __user *phandle; -+ -+ if (sizeof(*mem_import) != args_size) -+ goto bad_size; -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ phandle = compat_ptr(mem_import->phandle.compat_value); -+ else -+#endif -+ phandle = mem_import->phandle.value; -+ -+ if (mem_import->type == BASE_MEM_IMPORT_TYPE_INVALID) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ -+ if (kbase_mem_import(kctx, -+ (enum base_mem_import_type) -+ mem_import->type, -+ phandle, -+ 0, -+ &mem_import->gpu_va, -+ &mem_import->va_pages, -+ &mem_import->flags)) { -+ mem_import->type = BASE_MEM_IMPORT_TYPE_INVALID; -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ } -+ break; -+ } -+ case KBASE_FUNC_MEM_ALIAS: { -+ struct kbase_uk_mem_alias *alias = args; -+ struct base_mem_aliasing_info __user *user_ai; -+ struct base_mem_aliasing_info *ai; -+ -+ if (sizeof(*alias) != args_size) -+ goto bad_size; -+ -+ if (alias->nents > 2048) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ if (!alias->nents) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_ai = compat_ptr(alias->ai.compat_value); -+ else -+#endif -+ user_ai = alias->ai.value; -+ -+ ai = vmalloc(sizeof(*ai) * alias->nents); -+ -+ if (!ai) { -+ ukh->ret = MALI_ERROR_OUT_OF_MEMORY; -+ break; -+ } -+ -+ if (copy_from_user(ai, user_ai, -+ sizeof(*ai) * alias->nents)) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ goto copy_failed; -+ } -+ -+ alias->gpu_va = kbase_mem_alias(kctx, &alias->flags, -+ alias->stride, -+ alias->nents, ai, -+ &alias->va_pages); -+ if (!alias->gpu_va) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ goto no_alias; -+ } -+no_alias: -+copy_failed: -+ vfree(ai); -+ break; -+ } -+ case KBASE_FUNC_MEM_COMMIT: -+ { -+ struct kbase_uk_mem_commit *commit = args; -+ int ret; -+ -+ if (sizeof(*commit) != args_size) -+ goto bad_size; -+ -+ ret = kbase_mem_commit(kctx, commit->gpu_addr, -+ commit->pages); -+ -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ commit->result_subcode = -+ BASE_BACKING_THRESHOLD_ERROR_INVALID_ARGUMENTS; -+ -+ if (ret == 0) { -+ ukh->ret = MALI_ERROR_NONE; -+ commit->result_subcode = -+ BASE_BACKING_THRESHOLD_OK; -+ } else if (ret == -ENOMEM) { -+ commit->result_subcode = -+ BASE_BACKING_THRESHOLD_ERROR_OOM; -+ } -+ -+ break; -+ } -+ -+ case KBASE_FUNC_MEM_QUERY: -+ { -+ struct kbase_uk_mem_query *query = args; -+ -+ if (sizeof(*query) != args_size) -+ goto bad_size; -+ -+ if (kbase_mem_query(kctx, query->gpu_addr, -+ query->query, &query->value) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ else -+ ukh->ret = MALI_ERROR_NONE; -+ break; -+ } -+ break; -+ -+ case KBASE_FUNC_MEM_FLAGS_CHANGE: -+ { -+ struct kbase_uk_mem_flags_change *fc = args; -+ -+ if (sizeof(*fc) != args_size) -+ goto bad_size; -+ -+ if (kbase_mem_flags_change(kctx, fc->gpu_va, -+ fc->flags, fc->mask) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ -+ break; -+ } -+ case KBASE_FUNC_MEM_FREE: -+ { -+ struct kbase_uk_mem_free *mem = args; -+ -+ if (sizeof(*mem) != args_size) -+ goto bad_size; -+ -+ if (kbase_mem_free(kctx, mem->gpu_addr) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ -+ case KBASE_FUNC_JOB_SUBMIT: -+ { -+ struct kbase_uk_job_submit *job = args; -+ void __user *user_addr = NULL; -+ -+ if (sizeof(*job) != args_size) -+ goto bad_size; -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_addr = compat_ptr(job->addr.compat_value); -+ else -+#endif -+ user_addr = job->addr.value; -+ -+ if (kbase_jd_submit(kctx, user_addr, job->nr_atoms, -+ job->stride, false) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ -+#ifdef BASE_LEGACY_UK6_SUPPORT -+ case KBASE_FUNC_JOB_SUBMIT_UK6: -+ { -+ struct kbase_uk_job_submit *job = args; -+ void __user *user_addr = NULL; -+ -+ if (sizeof(*job) != args_size) -+ goto bad_size; -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_addr = compat_ptr(job->addr.compat_value); -+ else -+#endif -+ user_addr = job->addr.value; -+ -+ if (kbase_jd_submit(kctx, user_addr, job->nr_atoms, -+ job->stride, true) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+#endif -+ -+ case KBASE_FUNC_SYNC: -+ { -+ struct kbase_uk_sync_now *sn = args; -+ -+ if (sizeof(*sn) != args_size) -+ goto bad_size; -+ -+#ifndef CONFIG_MALI_COH_USER -+ if (kbase_sync_now(kctx, &sn->sset.basep_sset) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+#endif -+ break; -+ } -+ -+ case KBASE_FUNC_DISJOINT_QUERY: -+ { -+ struct kbase_uk_disjoint_query *dquery = args; -+ -+ if (sizeof(*dquery) != args_size) -+ goto bad_size; -+ -+ /* Get the disjointness counter value. */ -+ dquery->counter = kbase_disjoint_event_get(kctx->kbdev); -+ break; -+ } -+ -+ case KBASE_FUNC_POST_TERM: -+ { -+ kbase_event_close(kctx); -+ break; -+ } -+ -+ case KBASE_FUNC_HWCNT_SETUP: -+ { -+ struct kbase_uk_hwcnt_setup *setup = args; -+ -+ if (sizeof(*setup) != args_size) -+ goto bad_size; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ if (kbase_vinstr_legacy_hwc_setup(kbdev->vinstr_ctx, -+ &kctx->vinstr_cli, setup) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ break; -+ } -+ -+ case KBASE_FUNC_HWCNT_DUMP: -+ { -+ /* args ignored */ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ if (kbase_vinstr_hwc_dump(kctx->vinstr_cli, -+ BASE_HWCNT_READER_EVENT_MANUAL) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ break; -+ } -+ -+ case KBASE_FUNC_HWCNT_CLEAR: -+ { -+ /* args ignored */ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ if (kbase_vinstr_hwc_clear(kctx->vinstr_cli) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ break; -+ } -+ -+ case KBASE_FUNC_HWCNT_READER_SETUP: -+ { -+ struct kbase_uk_hwcnt_reader_setup *setup = args; -+ -+ if (sizeof(*setup) != args_size) -+ goto bad_size; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ if (kbase_vinstr_hwcnt_reader_setup(kbdev->vinstr_ctx, -+ setup) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ break; -+ } -+ -+ case KBASE_FUNC_GPU_PROPS_REG_DUMP: -+ { -+ struct kbase_uk_gpuprops *setup = args; -+ -+ if (sizeof(*setup) != args_size) -+ goto bad_size; -+ -+ if (kbase_gpuprops_uk_get_props(kctx, setup) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ case KBASE_FUNC_FIND_CPU_OFFSET: -+ { -+ struct kbase_uk_find_cpu_offset *find = args; -+ -+ if (sizeof(*find) != args_size) -+ goto bad_size; -+ -+ if (find->gpu_addr & ~PAGE_MASK) { -+ dev_warn(kbdev->dev, -+ "kbase_legacy_dispatch case KBASE_FUNC_FIND_CPU_OFFSET: find->gpu_addr: passed parameter is invalid"); -+ goto out_bad; -+ } -+ -+ if (find->size > SIZE_MAX || find->cpu_addr > ULONG_MAX) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ } else { -+ int err; -+ -+ err = kbasep_find_enclosing_cpu_mapping_offset( -+ kctx, -+ find->cpu_addr, -+ find->size, -+ &find->offset); -+ -+ if (err) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ } -+ break; -+ } -+ case KBASE_FUNC_GET_VERSION: -+ { -+ struct kbase_uk_get_ddk_version *get_version = (struct kbase_uk_get_ddk_version *)args; -+ -+ if (sizeof(*get_version) != args_size) -+ goto bad_size; -+ -+ /* version buffer size check is made in compile time assert */ -+ memcpy(get_version->version_buffer, -+ KERNEL_SIDE_DDK_VERSION_STRING, -+ sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); -+ get_version->version_string_size = -+ sizeof(KERNEL_SIDE_DDK_VERSION_STRING); -+ get_version->rk_version = ROCKCHIP_VERSION; -+ break; -+ } -+ -+ case KBASE_FUNC_STREAM_CREATE: -+ { -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ struct kbase_uk_stream_create *screate = (struct kbase_uk_stream_create *)args; -+ -+ if (sizeof(*screate) != args_size) -+ goto bad_size; -+ -+ if (strnlen(screate->name, sizeof(screate->name)) >= sizeof(screate->name)) { -+ /* not NULL terminated */ -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+ -+ if (kbase_sync_fence_stream_create(screate->name, -+ &screate->fd) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ else -+ ukh->ret = MALI_ERROR_NONE; -+#else /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+ break; -+ } -+ case KBASE_FUNC_FENCE_VALIDATE: -+ { -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ struct kbase_uk_fence_validate *fence_validate = (struct kbase_uk_fence_validate *)args; -+ -+ if (sizeof(*fence_validate) != args_size) -+ goto bad_size; -+ -+ if (kbase_sync_fence_validate(fence_validate->fd) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ else -+ ukh->ret = MALI_ERROR_NONE; -+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+ break; -+ } -+ -+ case KBASE_FUNC_SET_TEST_DATA: -+ { -+#if MALI_UNIT_TEST -+ struct kbase_uk_set_test_data *set_data = args; -+ -+ shared_kernel_test_data = set_data->test_data; -+ shared_kernel_test_data.kctx.value = (void __user *)kctx; -+ shared_kernel_test_data.mm.value = (void __user *)current->mm; -+ ukh->ret = MALI_ERROR_NONE; -+#endif /* MALI_UNIT_TEST */ -+ break; -+ } -+ -+ case KBASE_FUNC_INJECT_ERROR: -+ { -+#ifdef CONFIG_MALI_ERROR_INJECT -+ unsigned long flags; -+ struct kbase_error_params params = ((struct kbase_uk_error_params *)args)->params; -+ -+ /*mutex lock */ -+ spin_lock_irqsave(&kbdev->reg_op_lock, flags); -+ if (job_atom_inject_error(¶ms) != 0) -+ ukh->ret = MALI_ERROR_OUT_OF_MEMORY; -+ else -+ ukh->ret = MALI_ERROR_NONE; -+ spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); -+ /*mutex unlock */ -+#endif /* CONFIG_MALI_ERROR_INJECT */ -+ break; -+ } -+ -+ case KBASE_FUNC_MODEL_CONTROL: -+ { -+#ifdef CONFIG_MALI_NO_MALI -+ unsigned long flags; -+ struct kbase_model_control_params params = -+ ((struct kbase_uk_model_control_params *)args)->params; -+ -+ /*mutex lock */ -+ spin_lock_irqsave(&kbdev->reg_op_lock, flags); -+ if (gpu_model_control(kbdev->model, ¶ms) != 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ else -+ ukh->ret = MALI_ERROR_NONE; -+ spin_unlock_irqrestore(&kbdev->reg_op_lock, flags); -+ /*mutex unlock */ -+#endif /* CONFIG_MALI_NO_MALI */ -+ break; -+ } -+ -+#ifdef BASE_LEGACY_UK8_SUPPORT -+ case KBASE_FUNC_KEEP_GPU_POWERED: -+ { -+ dev_warn(kbdev->dev, "kbase_legacy_dispatch case KBASE_FUNC_KEEP_GPU_POWERED: function is deprecated and disabled\n"); -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ break; -+ } -+#endif /* BASE_LEGACY_UK8_SUPPORT */ -+ -+ case KBASE_FUNC_GET_PROFILING_CONTROLS: -+ { -+ struct kbase_uk_profiling_controls *controls = -+ (struct kbase_uk_profiling_controls *)args; -+ u32 i; -+ -+ if (sizeof(*controls) != args_size) -+ goto bad_size; -+ -+ for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) -+ controls->profiling_controls[i] = -+ kbdev->kbase_profiling_controls[i]; -+ -+ break; -+ } -+ -+ /* used only for testing purposes; these controls are to be set by gator through gator API */ -+ case KBASE_FUNC_SET_PROFILING_CONTROLS: -+ { -+ struct kbase_uk_profiling_controls *controls = -+ (struct kbase_uk_profiling_controls *)args; -+ u32 i; -+ -+ if (sizeof(*controls) != args_size) -+ goto bad_size; -+ -+ for (i = FBDUMP_CONTROL_MIN; i < FBDUMP_CONTROL_MAX; i++) -+ _mali_profiling_control(i, controls->profiling_controls[i]); -+ -+ break; -+ } -+ -+ case KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD: -+ { -+ struct kbase_uk_debugfs_mem_profile_add *add_data = -+ (struct kbase_uk_debugfs_mem_profile_add *)args; -+ char *buf; -+ char __user *user_buf; -+ -+ if (sizeof(*add_data) != args_size) -+ goto bad_size; -+ -+ if (add_data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { -+ dev_err(kbdev->dev, "buffer too big\n"); -+ goto out_bad; -+ } -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_buf = -+ compat_ptr(add_data->buf.compat_value); -+ else -+#endif -+ user_buf = add_data->buf.value; -+ -+ buf = kmalloc(add_data->len, GFP_KERNEL); -+ if (ZERO_OR_NULL_PTR(buf)) -+ goto out_bad; -+ -+ if (0 != copy_from_user(buf, user_buf, add_data->len)) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ kfree(buf); -+ goto out_bad; -+ } -+ -+ if (kbasep_mem_profile_debugfs_insert(kctx, buf, -+ add_data->len)) { -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ goto out_bad; -+ } -+ -+ break; -+ } -+ -+#ifdef CONFIG_MALI_NO_MALI -+ case KBASE_FUNC_SET_PRFCNT_VALUES: -+ { -+ -+ struct kbase_uk_prfcnt_values *params = -+ ((struct kbase_uk_prfcnt_values *)args); -+ gpu_model_set_dummy_prfcnt_sample(params->data, -+ params->size); -+ -+ break; -+ } -+#endif /* CONFIG_MALI_NO_MALI */ -+#ifdef BASE_LEGACY_UK10_4_SUPPORT -+ case KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4: -+ { -+ struct kbase_uk_tlstream_acquire_v10_4 *tlstream_acquire -+ = args; -+ int ret; -+ -+ if (sizeof(*tlstream_acquire) != args_size) -+ goto bad_size; -+ -+ ret = kbase_tlstream_acquire( -+ kctx, 0); -+ if (ret < 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ else -+ tlstream_acquire->fd = ret; -+ break; -+ } -+#endif /* BASE_LEGACY_UK10_4_SUPPORT */ -+ case KBASE_FUNC_TLSTREAM_ACQUIRE: -+ { -+ struct kbase_uk_tlstream_acquire *tlstream_acquire = -+ args; -+ int ret; -+ -+ if (sizeof(*tlstream_acquire) != args_size) -+ goto bad_size; -+ -+ if (tlstream_acquire->flags & ~BASE_TLSTREAM_FLAGS_MASK) -+ goto out_bad; -+ -+ ret = kbase_tlstream_acquire( -+ kctx, tlstream_acquire->flags); -+ if (ret < 0) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ else -+ tlstream_acquire->fd = ret; -+ break; -+ } -+ case KBASE_FUNC_TLSTREAM_FLUSH: -+ { -+ struct kbase_uk_tlstream_flush *tlstream_flush = -+ args; -+ -+ if (sizeof(*tlstream_flush) != args_size) -+ goto bad_size; -+ -+ kbase_tlstream_flush_streams(); -+ break; -+ } -+#if MALI_UNIT_TEST -+ case KBASE_FUNC_TLSTREAM_TEST: -+ { -+ struct kbase_uk_tlstream_test *tlstream_test = args; -+ -+ if (sizeof(*tlstream_test) != args_size) -+ goto bad_size; -+ -+ kbase_tlstream_test( -+ tlstream_test->tpw_count, -+ tlstream_test->msg_delay, -+ tlstream_test->msg_count, -+ tlstream_test->aux_msg); -+ break; -+ } -+ case KBASE_FUNC_TLSTREAM_STATS: -+ { -+ struct kbase_uk_tlstream_stats *tlstream_stats = args; -+ -+ if (sizeof(*tlstream_stats) != args_size) -+ goto bad_size; -+ -+ kbase_tlstream_stats( -+ &tlstream_stats->bytes_collected, -+ &tlstream_stats->bytes_generated); -+ break; -+ } -+#endif /* MALI_UNIT_TEST */ -+ -+ case KBASE_FUNC_GET_CONTEXT_ID: -+ { -+ struct kbase_uk_context_id *info = args; -+ -+ info->id = kctx->id; -+ break; -+ } -+ -+ case KBASE_FUNC_SOFT_EVENT_UPDATE: -+ { -+ struct kbase_uk_soft_event_update *update = args; -+ -+ if (sizeof(*update) != args_size) -+ goto bad_size; -+ -+ if (((update->new_status != BASE_JD_SOFT_EVENT_SET) && -+ (update->new_status != BASE_JD_SOFT_EVENT_RESET)) || -+ (update->flags != 0)) -+ goto out_bad; -+ -+ if (kbase_soft_event_update(kctx, update->evt, -+ update->new_status)) -+ ukh->ret = MALI_ERROR_FUNCTION_FAILED; -+ -+ break; -+ } -+ -+ default: -+ dev_err(kbdev->dev, "unknown ioctl %u\n", id); -+ goto out_bad; -+ } -+ -+ return ret; -+ -+bad_size: -+ dev_err(kbdev->dev, "Wrong syscall size (%d) for %08x\n", args_size, id); -+out_bad: -+ return -EINVAL; -+} -+ -+static struct kbase_device *to_kbase_device(struct device *dev) -+{ -+ return dev_get_drvdata(dev); -+} -+ -+static int assign_irqs(struct platform_device *pdev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); -+ int i; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ /* 3 IRQ resources */ -+ for (i = 0; i < 3; i++) { -+ struct resource *irq_res; -+ int irqtag; -+ -+ irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); -+ if (!irq_res) { -+ dev_err(kbdev->dev, "No IRQ resource at index %d\n", i); -+ return -ENOENT; -+ } -+ -+#ifdef CONFIG_OF -+ if (!strncasecmp(irq_res->name, "JOB", 3)) { -+ irqtag = JOB_IRQ_TAG; -+ } else if (!strncasecmp(irq_res->name, "MMU", 3)) { -+ irqtag = MMU_IRQ_TAG; -+ } else if (!strncasecmp(irq_res->name, "GPU", 3)) { -+ irqtag = GPU_IRQ_TAG; -+ } else { -+ dev_err(&pdev->dev, "Invalid irq res name: '%s'\n", -+ irq_res->name); -+ return -EINVAL; -+ } -+#else -+ irqtag = i; -+#endif /* CONFIG_OF */ -+ kbdev->irqs[irqtag].irq = irq_res->start; -+ kbdev->irqs[irqtag].flags = irq_res->flags & IRQF_TRIGGER_MASK; -+ } -+ -+ return 0; -+} -+ -+/* -+ * API to acquire device list mutex and -+ * return pointer to the device list head -+ */ -+const struct list_head *kbase_dev_list_get(void) -+{ -+ mutex_lock(&kbase_dev_list_lock); -+ return &kbase_dev_list; -+} -+KBASE_EXPORT_TEST_API(kbase_dev_list_get); -+ -+/* API to release the device list mutex */ -+void kbase_dev_list_put(const struct list_head *dev_list) -+{ -+ mutex_unlock(&kbase_dev_list_lock); -+} -+KBASE_EXPORT_TEST_API(kbase_dev_list_put); -+ -+/* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ -+struct kbase_device *kbase_find_device(int minor) -+{ -+ struct kbase_device *kbdev = NULL; -+ struct list_head *entry; -+ const struct list_head *dev_list = kbase_dev_list_get(); -+ -+ list_for_each(entry, dev_list) { -+ struct kbase_device *tmp; -+ -+ tmp = list_entry(entry, struct kbase_device, entry); -+ if (tmp->mdev.minor == minor || minor == -1) { -+ kbdev = tmp; -+ get_device(kbdev->dev); -+ break; -+ } -+ } -+ kbase_dev_list_put(dev_list); -+ -+ return kbdev; -+} -+EXPORT_SYMBOL(kbase_find_device); -+ -+void kbase_release_device(struct kbase_device *kbdev) -+{ -+ put_device(kbdev->dev); -+} -+EXPORT_SYMBOL(kbase_release_device); -+ -+#if KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE -+/* -+ * Older versions, before v4.6, of the kernel doesn't have -+ * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 -+ */ -+static int kstrtobool_from_user(const char __user *s, size_t count, bool *res) -+{ -+ char buf[32]; -+ -+ count = min(sizeof(buf), count); -+ -+ if (copy_from_user(buf, s, count)) -+ return -EFAULT; -+ buf[count] = '\0'; -+ -+ return strtobool(buf, res); -+} -+#endif -+ -+static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, size_t size, loff_t *off) -+{ -+ struct kbase_context *kctx = f->private_data; -+ int err; -+ bool value; -+ -+ err = kstrtobool_from_user(ubuf, size, &value); -+ if (err) -+ return err; -+ -+ if (value) -+ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); -+ else -+ kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); -+ -+ return size; -+} -+ -+static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) -+{ -+ struct kbase_context *kctx = f->private_data; -+ char buf[32]; -+ int count; -+ bool value; -+ -+ value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); -+ -+ count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); -+ -+ return simple_read_from_buffer(ubuf, size, off, buf, count); -+} -+ -+static const struct file_operations kbase_infinite_cache_fops = { -+ .open = simple_open, -+ .write = write_ctx_infinite_cache, -+ .read = read_ctx_infinite_cache, -+}; -+ -+static int kbase_open(struct inode *inode, struct file *filp) -+{ -+ struct kbase_device *kbdev = NULL; -+ struct kbase_context *kctx; -+ int ret = 0; -+#ifdef CONFIG_DEBUG_FS -+ char kctx_name[64]; -+#endif -+ -+ kbdev = kbase_find_device(iminor(inode)); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ kctx = kbase_create_context(kbdev, is_compat_task()); -+ if (!kctx) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ init_waitqueue_head(&kctx->event_queue); -+ filp->f_mode |= FMODE_UNSIGNED_OFFSET; -+ filp->private_data = kctx; -+ kctx->filp = filp; -+ -+ if (kbdev->infinite_cache_active_default) -+ kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); -+ -+#ifdef CONFIG_DEBUG_FS -+ snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); -+ -+ kctx->kctx_dentry = debugfs_create_dir(kctx_name, -+ kbdev->debugfs_ctx_directory); -+ -+ if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+#ifdef CONFIG_MALI_COH_USER -+ /* if cache is completely coherent at hardware level, then remove the -+ * infinite cache control support from debugfs. -+ */ -+#else -+ debugfs_create_file("infinite_cache", 0644, kctx->kctx_dentry, -+ kctx, &kbase_infinite_cache_fops); -+#endif /* CONFIG_MALI_COH_USER */ -+ -+ mutex_init(&kctx->mem_profile_lock); -+ -+ kbasep_jd_debugfs_ctx_init(kctx); -+ kbase_debug_mem_view_init(filp); -+ -+ kbase_debug_job_fault_context_init(kctx); -+ -+ kbase_mem_pool_debugfs_init(kctx->kctx_dentry, &kctx->mem_pool); -+ -+ kbase_jit_debugfs_init(kctx); -+#endif /* CONFIG_DEBUG_FS */ -+ -+ dev_dbg(kbdev->dev, "created base context\n"); -+ -+ { -+ struct kbasep_kctx_list_element *element; -+ -+ element = kzalloc(sizeof(*element), GFP_KERNEL); -+ if (element) { -+ mutex_lock(&kbdev->kctx_list_lock); -+ element->kctx = kctx; -+ list_add(&element->link, &kbdev->kctx_list); -+ KBASE_TLSTREAM_TL_NEW_CTX( -+ element->kctx, -+ (u32)(element->kctx->id), -+ (u32)(element->kctx->tgid)); -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } else { -+ /* we don't treat this as a fail - just warn about it */ -+ dev_warn(kbdev->dev, "couldn't add kctx to kctx_list\n"); -+ } -+ } -+ return 0; -+ -+ out: -+ kbase_release_device(kbdev); -+ return ret; -+} -+ -+static int kbase_release(struct inode *inode, struct file *filp) -+{ -+ struct kbase_context *kctx = filp->private_data; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbasep_kctx_list_element *element, *tmp; -+ bool found_element = false; -+ -+ KBASE_TLSTREAM_TL_DEL_CTX(kctx); -+ -+#ifdef CONFIG_DEBUG_FS -+ kbasep_mem_profile_debugfs_remove(kctx); -+ kbase_debug_job_fault_context_term(kctx); -+#endif -+ -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { -+ if (element->kctx == kctx) { -+ list_del(&element->link); -+ kfree(element); -+ found_element = true; -+ } -+ } -+ mutex_unlock(&kbdev->kctx_list_lock); -+ if (!found_element) -+ dev_warn(kbdev->dev, "kctx not in kctx_list\n"); -+ -+ filp->private_data = NULL; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ /* If this client was performing hwcnt dumping and did not explicitly -+ * detach itself, remove it from the vinstr core now */ -+ if (kctx->vinstr_cli) { -+ struct kbase_uk_hwcnt_setup setup; -+ -+ setup.dump_buffer = 0llu; -+ kbase_vinstr_legacy_hwc_setup( -+ kbdev->vinstr_ctx, &kctx->vinstr_cli, &setup); -+ } -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ -+ kbase_destroy_context(kctx); -+ -+ dev_dbg(kbdev->dev, "deleted base context\n"); -+ kbase_release_device(kbdev); -+ return 0; -+} -+ -+#define CALL_MAX_SIZE 536 -+ -+static long kbase_legacy_ioctl(struct file *filp, unsigned int cmd, -+ unsigned long arg) -+{ -+ u64 msg[(CALL_MAX_SIZE + 7) >> 3] = { 0xdeadbeefdeadbeefull }; /* alignment fixup */ -+ u32 size = _IOC_SIZE(cmd); -+ struct kbase_context *kctx = filp->private_data; -+ -+ if (size > CALL_MAX_SIZE) -+ return -ENOTTY; -+ -+ if (0 != copy_from_user(&msg, (void __user *)arg, size)) { -+ dev_err(kctx->kbdev->dev, "failed to copy ioctl argument into kernel space\n"); -+ return -EFAULT; -+ } -+ -+ if (kbase_legacy_dispatch(kctx, &msg, size) != 0) -+ return -EFAULT; -+ -+ if (0 != copy_to_user((void __user *)arg, &msg, size)) { -+ dev_err(kctx->kbdev->dev, "failed to copy results of UK call back to user space\n"); -+ return -EFAULT; -+ } -+ return 0; -+} -+ -+static int kbase_api_set_flags(struct kbase_context *kctx, -+ struct kbase_ioctl_set_flags *flags) -+{ -+ int err; -+ -+ /* setup pending, try to signal that we'll do the setup, -+ * if setup was already in progress, err this call -+ */ -+ if (atomic_cmpxchg(&kctx->setup_in_progress, 0, 1) != 0) -+ return -EINVAL; -+ -+ err = kbase_context_set_create_flags(kctx, flags->create_flags); -+ /* if bad flags, will stay stuck in setup mode */ -+ if (err) -+ return err; -+ -+ atomic_set(&kctx->setup_complete, 1); -+ return 0; -+} -+ -+static int kbase_api_job_submit(struct kbase_context *kctx, -+ struct kbase_ioctl_job_submit *submit) -+{ -+ void __user *user_addr = NULL; -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_addr = compat_ptr(submit->addr.compat_value); -+ else -+#endif -+ user_addr = submit->addr.value; -+ -+ return kbase_jd_submit(kctx, user_addr, submit->nr_atoms, -+ submit->stride, false); -+} -+ -+static int kbase_api_get_gpuprops(struct kbase_context *kctx, -+ struct kbase_ioctl_get_gpuprops *get_props) -+{ -+ struct kbase_gpu_props *kprops = &kctx->kbdev->gpu_props; -+ int err; -+ -+ if (get_props->flags != 0) { -+ dev_err(kctx->kbdev->dev, "Unsupported flags to get_gpuprops"); -+ return -EINVAL; -+ } -+ -+ if (get_props->size == 0) -+ return kprops->prop_buffer_size; -+ if (get_props->size < kprops->prop_buffer_size) -+ return -EINVAL; -+ -+ err = copy_to_user(get_props->buffer.value, kprops->prop_buffer, -+ kprops->prop_buffer_size); -+ if (err) -+ return err; -+ return kprops->prop_buffer_size; -+} -+ -+static int kbase_api_post_term(struct kbase_context *kctx) -+{ -+ kbase_event_close(kctx); -+ return 0; -+} -+ -+static int kbase_api_mem_alloc(struct kbase_context *kctx, -+ union kbase_ioctl_mem_alloc *alloc) -+{ -+ struct kbase_va_region *reg; -+ u64 flags = alloc->in.flags; -+ u64 gpu_va; -+ -+#if defined(CONFIG_64BIT) -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ /* force SAME_VA if a 64-bit client */ -+ flags |= BASE_MEM_SAME_VA; -+ } -+#endif -+ -+ reg = kbase_mem_alloc(kctx, alloc->in.va_pages, -+ alloc->in.commit_pages, -+ alloc->in.extent, -+ &flags, &gpu_va); -+ -+ if (!reg) -+ return -ENOMEM; -+ -+ alloc->out.flags = flags; -+ alloc->out.gpu_va = gpu_va; -+ -+ return 0; -+} -+ -+static int kbase_api_mem_query(struct kbase_context *kctx, -+ union kbase_ioctl_mem_query *query) -+{ -+ return kbase_mem_query(kctx, query->in.gpu_addr, -+ query->in.query, &query->out.value); -+} -+ -+static int kbase_api_mem_free(struct kbase_context *kctx, -+ struct kbase_ioctl_mem_free *free) -+{ -+ return kbase_mem_free(kctx, free->gpu_addr); -+} -+ -+static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, -+ struct kbase_ioctl_hwcnt_reader_setup *setup) -+{ -+ int ret; -+ struct kbase_uk_hwcnt_reader_setup args = { -+ .buffer_count = setup->buffer_count, -+ .jm_bm = setup->jm_bm, -+ .shader_bm = setup->shader_bm, -+ .tiler_bm = setup->tiler_bm, -+ .mmu_l2_bm = setup->mmu_l2_bm -+ }; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ ret = kbase_vinstr_hwcnt_reader_setup(kctx->kbdev->vinstr_ctx, &args); -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ -+ if (ret) -+ return ret; -+ return args.fd; -+} -+ -+static int kbase_api_hwcnt_enable(struct kbase_context *kctx, -+ struct kbase_ioctl_hwcnt_enable *enable) -+{ -+ int ret; -+ struct kbase_uk_hwcnt_setup args = { -+ .dump_buffer = enable->dump_buffer, -+ .jm_bm = enable->jm_bm, -+ .shader_bm = enable->shader_bm, -+ .tiler_bm = enable->tiler_bm, -+ .mmu_l2_bm = enable->mmu_l2_bm -+ }; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ ret = kbase_vinstr_legacy_hwc_setup(kctx->kbdev->vinstr_ctx, -+ &kctx->vinstr_cli, &args); -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ -+ return ret; -+} -+ -+static int kbase_api_hwcnt_dump(struct kbase_context *kctx) -+{ -+ int ret; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ ret = kbase_vinstr_hwc_dump(kctx->vinstr_cli, -+ BASE_HWCNT_READER_EVENT_MANUAL); -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ -+ return ret; -+} -+ -+static int kbase_api_hwcnt_clear(struct kbase_context *kctx) -+{ -+ int ret; -+ -+ mutex_lock(&kctx->vinstr_cli_lock); -+ ret = kbase_vinstr_hwc_clear(kctx->vinstr_cli); -+ mutex_unlock(&kctx->vinstr_cli_lock); -+ -+ return ret; -+} -+ -+static int kbase_api_disjoint_query(struct kbase_context *kctx, -+ struct kbase_ioctl_disjoint_query *query) -+{ -+ query->counter = kbase_disjoint_event_get(kctx->kbdev); -+ -+ return 0; -+} -+ -+static int kbase_api_get_ddk_version(struct kbase_context *kctx, -+ struct kbase_ioctl_get_ddk_version *version) -+{ -+ int ret; -+ int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); -+ -+ if (version->version_buffer.value == NULL) -+ return len; -+ -+ if (version->size < len) -+ return -EOVERFLOW; -+ -+ ret = copy_to_user(version->version_buffer.value, -+ KERNEL_SIDE_DDK_VERSION_STRING, -+ sizeof(KERNEL_SIDE_DDK_VERSION_STRING)); -+ -+ if (ret) -+ return ret; -+ -+ return len; -+} -+ -+static int kbase_api_mem_jit_init(struct kbase_context *kctx, -+ struct kbase_ioctl_mem_jit_init *jit_init) -+{ -+ return kbase_region_tracker_init_jit(kctx, jit_init->va_pages); -+} -+ -+static int kbase_api_mem_sync(struct kbase_context *kctx, -+ struct kbase_ioctl_mem_sync *sync) -+{ -+#ifdef CONFIG_MALI_COH_USER -+ return 0; -+#endif -+ struct basep_syncset sset = { -+ .mem_handle.basep.handle = sync->handle, -+ .user_addr = sync->user_addr, -+ .size = sync->size, -+ .type = sync->type -+ }; -+ -+ return kbase_sync_now(kctx, &sset); -+} -+ -+static int kbase_api_mem_find_cpu_offset(struct kbase_context *kctx, -+ union kbase_ioctl_mem_find_cpu_offset *find) -+{ -+ return kbasep_find_enclosing_cpu_mapping_offset( -+ kctx, -+ find->in.cpu_addr, -+ find->in.size, -+ &find->out.offset); -+} -+ -+static int kbase_api_get_context_id(struct kbase_context *kctx, -+ struct kbase_ioctl_get_context_id *info) -+{ -+ info->id = kctx->id; -+ -+ return 0; -+} -+ -+static int kbase_api_tlstream_acquire(struct kbase_context *kctx, -+ struct kbase_ioctl_tlstream_acquire *acquire) -+{ -+ return kbase_tlstream_acquire(kctx, acquire->flags); -+} -+ -+static int kbase_api_tlstream_flush(struct kbase_context *kctx) -+{ -+ kbase_tlstream_flush_streams(); -+ -+ return 0; -+} -+ -+static int kbase_api_mem_commit(struct kbase_context *kctx, -+ struct kbase_ioctl_mem_commit *commit) -+{ -+ return kbase_mem_commit(kctx, commit->gpu_addr, commit->pages); -+} -+ -+static int kbase_api_mem_alias(struct kbase_context *kctx, -+ union kbase_ioctl_mem_alias *alias) -+{ -+ struct base_mem_aliasing_info *ai; -+ void __user *user_addr = NULL; -+ u64 flags; -+ int err; -+ -+ if (alias->in.nents == 0 || alias->in.nents > 2048) -+ return -EINVAL; -+ -+ ai = vmalloc(sizeof(*ai) * alias->in.nents); -+ if (!ai) -+ return -ENOMEM; -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_addr = -+ compat_ptr(alias->in.aliasing_info.compat_value); -+ else -+#endif -+ user_addr = alias->in.aliasing_info.value; -+ -+ err = copy_from_user(ai, user_addr, sizeof(*ai) * alias->in.nents); -+ if (err) { -+ vfree(ai); -+ return err; -+ } -+ -+ flags = alias->in.flags; -+ -+ alias->out.gpu_va = kbase_mem_alias(kctx, &flags, -+ alias->in.stride, alias->in.nents, -+ ai, &alias->out.va_pages); -+ -+ alias->out.flags = flags; -+ -+ vfree(ai); -+ -+ if (alias->out.gpu_va == 0) -+ return -ENOMEM; -+ -+ return 0; -+} -+ -+static int kbase_api_mem_import(struct kbase_context *kctx, -+ union kbase_ioctl_mem_import *import) -+{ -+ int ret; -+ u64 flags = import->in.flags; -+ void __user *phandle; -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ phandle = compat_ptr(import->in.phandle.compat_value); -+ else -+#endif -+ phandle = import->in.phandle.value; -+ -+ ret = kbase_mem_import(kctx, -+ import->in.type, -+ phandle, -+ import->in.padding, -+ &import->out.gpu_va, -+ &import->out.va_pages, -+ &flags); -+ -+ import->out.flags = flags; -+ -+ return ret; -+} -+ -+static int kbase_api_mem_flags_change(struct kbase_context *kctx, -+ struct kbase_ioctl_mem_flags_change *change) -+{ -+ return kbase_mem_flags_change(kctx, change->gpu_va, -+ change->flags, change->mask); -+} -+ -+static int kbase_api_stream_create(struct kbase_context *kctx, -+ struct kbase_ioctl_stream_create *stream) -+{ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ int fd, ret; -+ -+ /* Name must be NULL-terminated and padded with NULLs, so check last -+ * character is NULL -+ */ -+ if (stream->name[sizeof(stream->name)-1] != 0) -+ return -EINVAL; -+ -+ ret = kbase_sync_fence_stream_create(stream->name, &fd); -+ -+ if (ret) -+ return ret; -+ return fd; -+#else -+ return -ENOENT; -+#endif -+} -+ -+static int kbase_api_fence_validate(struct kbase_context *kctx, -+ struct kbase_ioctl_fence_validate *validate) -+{ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ return kbase_sync_fence_validate(validate->fd); -+#else -+ return -ENOENT; -+#endif -+} -+ -+static int kbase_api_get_profiling_controls(struct kbase_context *kctx, -+ struct kbase_ioctl_get_profiling_controls *controls) -+{ -+ if (controls->count > FBDUMP_CONTROL_MAX) -+ return -EINVAL; -+ -+ return copy_to_user(controls->buffer.value, -+ &kctx->kbdev->kbase_profiling_controls[ -+ FBDUMP_CONTROL_MIN], -+ controls->count * sizeof(u32)); -+} -+ -+static int kbase_api_mem_profile_add(struct kbase_context *kctx, -+ struct kbase_ioctl_mem_profile_add *data) -+{ -+ char __user *user_buf; -+ char *buf; -+ int err; -+ -+ if (data->len > KBASE_MEM_PROFILE_MAX_BUF_SIZE) { -+ dev_err(kctx->kbdev->dev, "mem_profile_add: buffer too big\n"); -+ return -EINVAL; -+ } -+ -+ buf = kmalloc(data->len, GFP_KERNEL); -+ if (ZERO_OR_NULL_PTR(buf)) -+ return -ENOMEM; -+ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ user_buf = compat_ptr(data->buffer.compat_value); -+ else -+#endif -+ user_buf = data->buffer.value; -+ -+ err = copy_from_user(buf, user_buf, data->len); -+ if (err) { -+ kfree(buf); -+ return err; -+ } -+ -+ return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); -+} -+ -+static int kbase_api_soft_event_update(struct kbase_context *kctx, -+ struct kbase_ioctl_soft_event_update *update) -+{ -+ if (update->flags != 0) -+ return -EINVAL; -+ -+ return kbase_soft_event_update(kctx, update->event, update->new_status); -+} -+ -+#if MALI_UNIT_TEST -+static int kbase_api_tlstream_test(struct kbase_context *kctx, -+ struct kbase_ioctl_tlstream_test *test) -+{ -+ kbase_tlstream_test( -+ test->tpw_count, -+ test->msg_delay, -+ test->msg_count, -+ test->aux_msg); -+ -+ return 0; -+} -+ -+static int kbase_api_tlstream_stats(struct kbase_context *kctx, -+ struct kbase_ioctl_tlstream_stats *stats) -+{ -+ kbase_tlstream_stats( -+ &stats->bytes_collected, -+ &stats->bytes_generated); -+ -+ return 0; -+} -+#endif /* MALI_UNIT_TEST */ -+ -+#define KBASE_HANDLE_IOCTL(cmd, function) \ -+ case cmd: \ -+ do { \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ -+ return function(kctx); \ -+ } while (0) -+ -+#define KBASE_HANDLE_IOCTL_IN(cmd, function, type) \ -+ case cmd: \ -+ do { \ -+ type param; \ -+ int err; \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ -+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -+ err = copy_from_user(¶m, uarg, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ return function(kctx, ¶m); \ -+ } while (0) -+ -+#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type) \ -+ case cmd: \ -+ do { \ -+ type param; \ -+ int ret, err; \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ -+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -+ ret = function(kctx, ¶m); \ -+ err = copy_to_user(uarg, ¶m, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ return ret; \ -+ } while (0) -+ -+#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type) \ -+ case cmd: \ -+ do { \ -+ type param; \ -+ int ret, err; \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ -+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -+ err = copy_from_user(¶m, uarg, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ ret = function(kctx, ¶m); \ -+ err = copy_to_user(uarg, ¶m, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ return ret; \ -+ } while (0) -+ -+static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -+{ -+ struct kbase_context *kctx = filp->private_data; -+ struct kbase_device *kbdev = kctx->kbdev; -+ void __user *uarg = (void __user *)arg; -+ -+ /* The UK ioctl values overflow the cmd field causing the type to be -+ * incremented -+ */ -+ if (_IOC_TYPE(cmd) == LINUX_UK_BASE_MAGIC+2) -+ return kbase_legacy_ioctl(filp, cmd, arg); -+ -+ /* The UK version check IOCTL doesn't overflow the cmd field, so is -+ * handled separately here -+ */ -+ if (cmd == _IOC(_IOC_READ|_IOC_WRITE, LINUX_UK_BASE_MAGIC, -+ UKP_FUNC_ID_CHECK_VERSION, -+ sizeof(struct uku_version_check_args))) -+ return kbase_legacy_ioctl(filp, cmd, arg); -+ -+ /* Only these ioctls are available until setup is complete */ -+ switch (cmd) { -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK, -+ kbase_api_handshake, -+ struct kbase_ioctl_version_check); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, -+ kbase_api_set_flags, -+ struct kbase_ioctl_set_flags); -+ } -+ -+ /* Block call until version handshake and setup is complete */ -+ if (kctx->api_version == 0 || !atomic_read(&kctx->setup_complete)) -+ return -EINVAL; -+ -+ /* Normal ioctls */ -+ switch (cmd) { -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, -+ kbase_api_job_submit, -+ struct kbase_ioctl_job_submit); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, -+ kbase_api_get_gpuprops, -+ struct kbase_ioctl_get_gpuprops); -+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, -+ kbase_api_post_term); -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, -+ kbase_api_mem_alloc, -+ union kbase_ioctl_mem_alloc); -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, -+ kbase_api_mem_query, -+ union kbase_ioctl_mem_query); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FREE, -+ kbase_api_mem_free, -+ struct kbase_ioctl_mem_free); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, -+ kbase_api_hwcnt_reader_setup, -+ struct kbase_ioctl_hwcnt_reader_setup); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_ENABLE, -+ kbase_api_hwcnt_enable, -+ struct kbase_ioctl_hwcnt_enable); -+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_DUMP, -+ kbase_api_hwcnt_dump); -+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_HWCNT_CLEAR, -+ kbase_api_hwcnt_clear); -+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_DISJOINT_QUERY, -+ kbase_api_disjoint_query, -+ struct kbase_ioctl_disjoint_query); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_DDK_VERSION, -+ kbase_api_get_ddk_version, -+ struct kbase_ioctl_get_ddk_version); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_JIT_INIT, -+ kbase_api_mem_jit_init, -+ struct kbase_ioctl_mem_jit_init); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_SYNC, -+ kbase_api_mem_sync, -+ struct kbase_ioctl_mem_sync); -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_FIND_CPU_OFFSET, -+ kbase_api_mem_find_cpu_offset, -+ union kbase_ioctl_mem_find_cpu_offset); -+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_GET_CONTEXT_ID, -+ kbase_api_get_context_id, -+ struct kbase_ioctl_get_context_id); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_ACQUIRE, -+ kbase_api_tlstream_acquire, -+ struct kbase_ioctl_tlstream_acquire); -+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_TLSTREAM_FLUSH, -+ kbase_api_tlstream_flush); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_COMMIT, -+ kbase_api_mem_commit, -+ struct kbase_ioctl_mem_commit); -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALIAS, -+ kbase_api_mem_alias, -+ union kbase_ioctl_mem_alias); -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_IMPORT, -+ kbase_api_mem_import, -+ union kbase_ioctl_mem_import); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_FLAGS_CHANGE, -+ kbase_api_mem_flags_change, -+ struct kbase_ioctl_mem_flags_change); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STREAM_CREATE, -+ kbase_api_stream_create, -+ struct kbase_ioctl_stream_create); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_FENCE_VALIDATE, -+ kbase_api_fence_validate, -+ struct kbase_ioctl_fence_validate); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_PROFILING_CONTROLS, -+ kbase_api_get_profiling_controls, -+ struct kbase_ioctl_get_profiling_controls); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_MEM_PROFILE_ADD, -+ kbase_api_mem_profile_add, -+ struct kbase_ioctl_mem_profile_add); -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, -+ kbase_api_soft_event_update, -+ struct kbase_ioctl_soft_event_update); -+ -+#if MALI_UNIT_TEST -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, -+ kbase_api_tlstream_test, -+ struct kbase_ioctl_tlstream_test); -+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, -+ kbase_api_tlstream_stats, -+ struct kbase_ioctl_tlstream_stats); -+#endif -+ } -+ -+ dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); -+ -+ return -ENOIOCTLCMD; -+} -+ -+static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) -+{ -+ struct kbase_context *kctx = filp->private_data; -+ struct base_jd_event_v2 uevent; -+ int out_count = 0; -+ -+ if (count < sizeof(uevent)) -+ return -ENOBUFS; -+ -+ do { -+ while (kbase_event_dequeue(kctx, &uevent)) { -+ if (out_count > 0) -+ goto out; -+ -+ if (filp->f_flags & O_NONBLOCK) -+ return -EAGAIN; -+ -+ if (wait_event_interruptible(kctx->event_queue, -+ kbase_event_pending(kctx)) != 0) -+ return -ERESTARTSYS; -+ } -+ if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { -+ if (out_count == 0) -+ return -EPIPE; -+ goto out; -+ } -+ -+ if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) -+ return -EFAULT; -+ -+ buf += sizeof(uevent); -+ out_count++; -+ count -= sizeof(uevent); -+ } while (count >= sizeof(uevent)); -+ -+ out: -+ return out_count * sizeof(uevent); -+} -+ -+static unsigned int kbase_poll(struct file *filp, poll_table *wait) -+{ -+ struct kbase_context *kctx = filp->private_data; -+ -+ poll_wait(filp, &kctx->event_queue, wait); -+ if (kbase_event_pending(kctx)) -+ return POLLIN | POLLRDNORM; -+ -+ return 0; -+} -+ -+void kbase_event_wakeup(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ wake_up_interruptible(&kctx->event_queue); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_event_wakeup); -+ -+static int kbase_check_flags(int flags) -+{ -+ /* Enforce that the driver keeps the O_CLOEXEC flag so that execve() always -+ * closes the file descriptor in a child process. -+ */ -+ if (0 == (flags & O_CLOEXEC)) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+ -+/** -+ * align_and_check - Align the specified pointer to the provided alignment and -+ * check that it is still in range. -+ * @gap_end: Highest possible start address for allocation (end of gap in -+ * address space) -+ * @gap_start: Start address of current memory area / gap in address space -+ * @info: vm_unmapped_area_info structure passed to caller, containing -+ * alignment, length and limits for the allocation -+ * @is_shader_code: True if the allocation is for shader code (which has -+ * additional alignment requirements) -+ * -+ * Return: true if gap_end is now aligned correctly and is still in range, -+ * false otherwise -+ */ -+static bool align_and_check(unsigned long *gap_end, unsigned long gap_start, -+ struct vm_unmapped_area_info *info, bool is_shader_code) -+{ -+ /* Compute highest gap address at the desired alignment */ -+ (*gap_end) -= info->length; -+ (*gap_end) -= (*gap_end - info->align_offset) & info->align_mask; -+ -+ if (is_shader_code) { -+ /* Check for 4GB boundary */ -+ if (0 == (*gap_end & BASE_MEM_MASK_4GB)) -+ (*gap_end) -= (info->align_offset ? info->align_offset : -+ info->length); -+ if (0 == ((*gap_end + info->length) & BASE_MEM_MASK_4GB)) -+ (*gap_end) -= (info->align_offset ? info->align_offset : -+ info->length); -+ -+ if (!(*gap_end & BASE_MEM_MASK_4GB) || !((*gap_end + -+ info->length) & BASE_MEM_MASK_4GB)) -+ return false; -+ } -+ -+ -+ if ((*gap_end < info->low_limit) || (*gap_end < gap_start)) -+ return false; -+ -+ -+ return true; -+} -+ -+/* The following function is taken from the kernel and just -+ * renamed. As it's not exported to modules we must copy-paste it here. -+ */ -+ -+static unsigned long kbase_unmapped_area_topdown(struct vm_unmapped_area_info -+ *info, bool is_shader_code) -+{ -+ struct mm_struct *mm = current->mm; -+ struct vm_area_struct *vma; -+ unsigned long length, low_limit, high_limit, gap_start, gap_end; -+ -+ /* Adjust search length to account for worst case alignment overhead */ -+ length = info->length + info->align_mask; -+ if (length < info->length) -+ return -ENOMEM; -+ -+ /* -+ * Adjust search limits by the desired length. -+ * See implementation comment at top of unmapped_area(). -+ */ -+ gap_end = info->high_limit; -+ if (gap_end < length) -+ return -ENOMEM; -+ high_limit = gap_end - length; -+ -+ if (info->low_limit > high_limit) -+ return -ENOMEM; -+ low_limit = info->low_limit + length; -+ -+ /* Check highest gap, which does not precede any rbtree node */ -+ gap_start = mm->highest_vm_end; -+ if (gap_start <= high_limit) { -+ if (align_and_check(&gap_end, gap_start, info, is_shader_code)) -+ return gap_end; -+ } -+ -+ /* Check if rbtree root looks promising */ -+ if (RB_EMPTY_ROOT(&mm->mm_rb)) -+ return -ENOMEM; -+ vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); -+ if (vma->rb_subtree_gap < length) -+ return -ENOMEM; -+ -+ while (true) { -+ /* Visit right subtree if it looks promising */ -+ gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0; -+ if (gap_start <= high_limit && vma->vm_rb.rb_right) { -+ struct vm_area_struct *right = -+ rb_entry(vma->vm_rb.rb_right, -+ struct vm_area_struct, vm_rb); -+ if (right->rb_subtree_gap >= length) { -+ vma = right; -+ continue; -+ } -+ } -+ -+check_current: -+ /* Check if current node has a suitable gap */ -+ gap_end = vma->vm_start; -+ if (gap_end < low_limit) -+ return -ENOMEM; -+ if (gap_start <= high_limit && gap_end - gap_start >= length) { -+ /* We found a suitable gap. Clip it with the original -+ * high_limit. */ -+ if (gap_end > info->high_limit) -+ gap_end = info->high_limit; -+ -+ if (align_and_check(&gap_end, gap_start, info, -+ is_shader_code)) -+ return gap_end; -+ } -+ -+ /* Visit left subtree if it looks promising */ -+ if (vma->vm_rb.rb_left) { -+ struct vm_area_struct *left = -+ rb_entry(vma->vm_rb.rb_left, -+ struct vm_area_struct, vm_rb); -+ if (left->rb_subtree_gap >= length) { -+ vma = left; -+ continue; -+ } -+ } -+ -+ /* Go back up the rbtree to find next candidate node */ -+ while (true) { -+ struct rb_node *prev = &vma->vm_rb; -+ if (!rb_parent(prev)) -+ return -ENOMEM; -+ vma = rb_entry(rb_parent(prev), -+ struct vm_area_struct, vm_rb); -+ if (prev == vma->vm_rb.rb_right) { -+ gap_start = vma->vm_prev ? -+ vma->vm_prev->vm_end : 0; -+ goto check_current; -+ } -+ } -+ } -+ -+ return -ENOMEM; -+} -+ -+static unsigned long kbase_get_unmapped_area(struct file *filp, -+ const unsigned long addr, const unsigned long len, -+ const unsigned long pgoff, const unsigned long flags) -+{ -+ /* based on get_unmapped_area, but simplified slightly due to that some -+ * values are known in advance */ -+ struct kbase_context *kctx = filp->private_data; -+ struct mm_struct *mm = current->mm; -+ struct vm_unmapped_area_info info; -+ unsigned long align_offset = 0; -+ unsigned long align_mask = 0; -+ unsigned long high_limit = mm->mmap_base; -+ unsigned long low_limit = PAGE_SIZE; -+ int cpu_va_bits = BITS_PER_LONG; -+ int gpu_pc_bits = -+ kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; -+ bool is_shader_code = false; -+ unsigned long ret; -+ -+ /* err on fixed address */ -+ if ((flags & MAP_FIXED) || addr) -+ return -EINVAL; -+ -+#ifdef CONFIG_64BIT -+ /* too big? */ -+ if (len > TASK_SIZE - SZ_2M) -+ return -ENOMEM; -+ -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ -+ if (kbase_hw_has_feature(kctx->kbdev, -+ BASE_HW_FEATURE_33BIT_VA)) { -+ high_limit = kctx->same_va_end << PAGE_SHIFT; -+ } else { -+ high_limit = min_t(unsigned long, mm->mmap_base, -+ (kctx->same_va_end << PAGE_SHIFT)); -+ if (len >= SZ_2M) { -+ align_offset = SZ_2M; -+ align_mask = SZ_2M - 1; -+ } -+ } -+ -+ low_limit = SZ_2M; -+ } else { -+ cpu_va_bits = 32; -+ } -+#endif /* CONFIG_64BIT */ -+ if ((PFN_DOWN(BASE_MEM_COOKIE_BASE) <= pgoff) && -+ (PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) > pgoff)) { -+ int cookie = pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); -+ -+ if (!kctx->pending_regions[cookie]) -+ return -EINVAL; -+ -+ if (!(kctx->pending_regions[cookie]->flags & -+ KBASE_REG_GPU_NX)) { -+ if (cpu_va_bits > gpu_pc_bits) { -+ align_offset = 1ULL << gpu_pc_bits; -+ align_mask = align_offset - 1; -+ is_shader_code = true; -+ } -+ } -+#ifndef CONFIG_64BIT -+ } else { -+ return current->mm->get_unmapped_area(filp, addr, len, pgoff, -+ flags); -+#endif -+ } -+ -+ info.flags = 0; -+ info.length = len; -+ info.low_limit = low_limit; -+ info.high_limit = high_limit; -+ info.align_offset = align_offset; -+ info.align_mask = align_mask; -+ -+ ret = kbase_unmapped_area_topdown(&info, is_shader_code); -+ -+ if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && -+ high_limit < (kctx->same_va_end << PAGE_SHIFT)) { -+ /* Retry above mmap_base */ -+ info.low_limit = mm->mmap_base; -+ info.high_limit = min_t(u64, TASK_SIZE, -+ (kctx->same_va_end << PAGE_SHIFT)); -+ -+ ret = kbase_unmapped_area_topdown(&info, is_shader_code); -+ } -+ -+ return ret; -+} -+ -+static const struct file_operations kbase_fops = { -+ .owner = THIS_MODULE, -+ .open = kbase_open, -+ .release = kbase_release, -+ .read = kbase_read, -+ .poll = kbase_poll, -+ .unlocked_ioctl = kbase_ioctl, -+ .compat_ioctl = kbase_ioctl, -+ .mmap = kbase_mmap, -+ .check_flags = kbase_check_flags, -+ .get_unmapped_area = kbase_get_unmapped_area, -+}; -+ -+#ifndef CONFIG_MALI_NO_MALI -+void kbase_os_reg_write(struct kbase_device *kbdev, u16 offset, u32 value) -+{ -+ writel(value, kbdev->reg + offset); -+} -+ -+u32 kbase_os_reg_read(struct kbase_device *kbdev, u16 offset) -+{ -+ return readl(kbdev->reg + offset); -+} -+#endif /* !CONFIG_MALI_NO_MALI */ -+ -+/** -+ * show_policy - Show callback for the power_policy sysfs file. -+ * -+ * This function is called to get the contents of the power_policy sysfs -+ * file. This is a list of the available policies with the currently active one -+ * surrounded by square brackets. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) -+{ -+ struct kbase_device *kbdev; -+ const struct kbase_pm_policy *current_policy; -+ const struct kbase_pm_policy *const *policy_list; -+ int policy_count; -+ int i; -+ ssize_t ret = 0; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ current_policy = kbase_pm_get_policy(kbdev); -+ -+ policy_count = kbase_pm_list_policies(&policy_list); -+ -+ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { -+ if (policy_list[i] == current_policy) -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); -+ else -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); -+ } -+ -+ if (ret < PAGE_SIZE - 1) { -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); -+ } else { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/** -+ * set_policy - Store callback for the power_policy sysfs file. -+ * -+ * This function is called when the power_policy sysfs file is written to. -+ * It matches the requested policy against the available policies and if a -+ * matching policy is found calls kbase_pm_set_policy() to change the -+ * policy. -+ * -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ const struct kbase_pm_policy *new_policy = NULL; -+ const struct kbase_pm_policy *const *policy_list; -+ int policy_count; -+ int i; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ policy_count = kbase_pm_list_policies(&policy_list); -+ -+ for (i = 0; i < policy_count; i++) { -+ if (sysfs_streq(policy_list[i]->name, buf)) { -+ new_policy = policy_list[i]; -+ break; -+ } -+ } -+ -+ if (!new_policy) { -+ dev_err(dev, "power_policy: policy not found\n"); -+ return -EINVAL; -+ } -+ -+ kbase_pm_set_policy(kbdev, new_policy); -+ -+ return count; -+} -+ -+/* -+ * The sysfs file power_policy. -+ * -+ * This is used for obtaining information about the available policies, -+ * determining which policy is currently active, and changing the active -+ * policy. -+ */ -+static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); -+ -+/** -+ * show_ca_policy - Show callback for the core_availability_policy sysfs file. -+ * -+ * This function is called to get the contents of the core_availability_policy -+ * sysfs file. This is a list of the available policies with the currently -+ * active one surrounded by square brackets. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_ca_policy(struct device *dev, struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ const struct kbase_pm_ca_policy *current_policy; -+ const struct kbase_pm_ca_policy *const *policy_list; -+ int policy_count; -+ int i; -+ ssize_t ret = 0; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ current_policy = kbase_pm_ca_get_policy(kbdev); -+ -+ policy_count = kbase_pm_ca_list_policies(&policy_list); -+ -+ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { -+ if (policy_list[i] == current_policy) -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); -+ else -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); -+ } -+ -+ if (ret < PAGE_SIZE - 1) { -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); -+ } else { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/** -+ * set_ca_policy - Store callback for the core_availability_policy sysfs file. -+ * -+ * This function is called when the core_availability_policy sysfs file is -+ * written to. It matches the requested policy against the available policies -+ * and if a matching policy is found calls kbase_pm_set_policy() to change -+ * the policy. -+ * -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_ca_policy(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ const struct kbase_pm_ca_policy *new_policy = NULL; -+ const struct kbase_pm_ca_policy *const *policy_list; -+ int policy_count; -+ int i; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ policy_count = kbase_pm_ca_list_policies(&policy_list); -+ -+ for (i = 0; i < policy_count; i++) { -+ if (sysfs_streq(policy_list[i]->name, buf)) { -+ new_policy = policy_list[i]; -+ break; -+ } -+ } -+ -+ if (!new_policy) { -+ dev_err(dev, "core_availability_policy: policy not found\n"); -+ return -EINVAL; -+ } -+ -+ kbase_pm_ca_set_policy(kbdev, new_policy); -+ -+ return count; -+} -+ -+/* -+ * The sysfs file core_availability_policy -+ * -+ * This is used for obtaining information about the available policies, -+ * determining which policy is currently active, and changing the active -+ * policy. -+ */ -+static DEVICE_ATTR(core_availability_policy, S_IRUGO | S_IWUSR, show_ca_policy, set_ca_policy); -+ -+/* -+ * show_core_mask - Show callback for the core_mask sysfs file. -+ * -+ * This function is called to get the contents of the core_mask sysfs file. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret = 0; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Current core mask (JS0) : 0x%llX\n", -+ kbdev->pm.debug_core_mask[0]); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Current core mask (JS1) : 0x%llX\n", -+ kbdev->pm.debug_core_mask[1]); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Current core mask (JS2) : 0x%llX\n", -+ kbdev->pm.debug_core_mask[2]); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Available core mask : 0x%llX\n", -+ kbdev->gpu_props.props.raw_props.shader_present); -+ -+ return ret; -+} -+ -+/** -+ * set_core_mask - Store callback for the core_mask sysfs file. -+ * -+ * This function is called when the core_mask sysfs file is written to. -+ * -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ u64 new_core_mask[3]; -+ int items; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ items = sscanf(buf, "%llx %llx %llx", -+ &new_core_mask[0], &new_core_mask[1], -+ &new_core_mask[2]); -+ -+ if (items == 1) -+ new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; -+ -+ if (items == 1 || items == 3) { -+ u64 shader_present = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ u64 group0_core_mask = -+ kbdev->gpu_props.props.coherency_info.group[0]. -+ core_mask; -+ -+ if ((new_core_mask[0] & shader_present) != new_core_mask[0] || -+ !(new_core_mask[0] & group0_core_mask) || -+ (new_core_mask[1] & shader_present) != -+ new_core_mask[1] || -+ !(new_core_mask[1] & group0_core_mask) || -+ (new_core_mask[2] & shader_present) != -+ new_core_mask[2] || -+ !(new_core_mask[2] & group0_core_mask)) { -+ dev_err(dev, "power_policy: invalid core specification\n"); -+ return -EINVAL; -+ } -+ -+ if (kbdev->pm.debug_core_mask[0] != new_core_mask[0] || -+ kbdev->pm.debug_core_mask[1] != -+ new_core_mask[1] || -+ kbdev->pm.debug_core_mask[2] != -+ new_core_mask[2]) { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], -+ new_core_mask[1], new_core_mask[2]); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+ -+ return count; -+ } -+ -+ dev_err(kbdev->dev, "Couldn't process set_core_mask write operation.\n" -+ "Use format \n" -+ "or \n"); -+ return -EINVAL; -+} -+ -+/* -+ * The sysfs file core_mask. -+ * -+ * This is used to restrict shader core availability for debugging purposes. -+ * Reading it will show the current core mask and the mask of cores available. -+ * Writing to it will set the current core mask. -+ */ -+static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); -+ -+/** -+ * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs -+ * file. -+ * -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The value written to the sysfs file. -+ * @count: The number of bytes written to the sysfs file. -+ * -+ * This allows setting the timeout for software jobs. Waiting soft event wait -+ * jobs will be cancelled after this period expires, while soft fence wait jobs -+ * will print debug information if the fence debug feature is enabled. -+ * -+ * This is expressed in milliseconds. -+ * -+ * Return: count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_soft_job_timeout(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int soft_job_timeout_ms; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || -+ (soft_job_timeout_ms <= 0)) -+ return -EINVAL; -+ -+ atomic_set(&kbdev->js_data.soft_job_timeout_ms, -+ soft_job_timeout_ms); -+ -+ return count; -+} -+ -+/** -+ * show_soft_job_timeout - Show callback for the soft_job_timeout sysfs -+ * file. -+ * -+ * This will return the timeout for the software jobs. -+ * -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer for the sysfs file contents. -+ * -+ * Return: The number of bytes output to buf. -+ */ -+static ssize_t show_soft_job_timeout(struct device *dev, -+ struct device_attribute *attr, -+ char * const buf) -+{ -+ struct kbase_device *kbdev; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ return scnprintf(buf, PAGE_SIZE, "%i\n", -+ atomic_read(&kbdev->js_data.soft_job_timeout_ms)); -+} -+ -+static DEVICE_ATTR(soft_job_timeout, S_IRUGO | S_IWUSR, -+ show_soft_job_timeout, set_soft_job_timeout); -+ -+static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, -+ int default_ticks, u32 old_ticks) -+{ -+ if (timeout_ms > 0) { -+ u64 ticks = timeout_ms * 1000000ULL; -+ do_div(ticks, kbdev->js_data.scheduling_period_ns); -+ if (!ticks) -+ return 1; -+ return ticks; -+ } else if (timeout_ms < 0) { -+ return default_ticks; -+ } else { -+ return old_ticks; -+ } -+} -+ -+/** -+ * set_js_timeouts - Store callback for the js_timeouts sysfs file. -+ * -+ * This function is called to get the contents of the js_timeouts sysfs -+ * file. This file contains five values separated by whitespace. The values -+ * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, -+ * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING -+ * configuration values (in that order), with the difference that the js_timeout -+ * values are expressed in MILLISECONDS. -+ * -+ * The js_timeouts sysfile file allows the current values in -+ * use by the job scheduler to get override. Note that a value needs to -+ * be other than 0 for it to override the current job scheduler value. -+ * -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int items; -+ long js_soft_stop_ms; -+ long js_soft_stop_ms_cl; -+ long js_hard_stop_ms_ss; -+ long js_hard_stop_ms_cl; -+ long js_hard_stop_ms_dumping; -+ long js_reset_ms_ss; -+ long js_reset_ms_cl; -+ long js_reset_ms_dumping; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", -+ &js_soft_stop_ms, &js_soft_stop_ms_cl, -+ &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, -+ &js_hard_stop_ms_dumping, &js_reset_ms_ss, -+ &js_reset_ms_cl, &js_reset_ms_dumping); -+ -+ if (items == 8) { -+ struct kbasep_js_device_data *js_data = &kbdev->js_data; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+#define UPDATE_TIMEOUT(ticks_name, ms_name, default) do {\ -+ js_data->ticks_name = timeout_ms_to_ticks(kbdev, ms_name, \ -+ default, js_data->ticks_name); \ -+ dev_dbg(kbdev->dev, "Overriding " #ticks_name \ -+ " with %lu ticks (%lu ms)\n", \ -+ (unsigned long)js_data->ticks_name, \ -+ ms_name); \ -+ } while (0) -+ -+ UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, -+ DEFAULT_JS_SOFT_STOP_TICKS); -+ UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, -+ DEFAULT_JS_SOFT_STOP_TICKS_CL); -+ UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? -+ DEFAULT_JS_HARD_STOP_TICKS_SS_8408 : -+ DEFAULT_JS_HARD_STOP_TICKS_SS); -+ UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, -+ DEFAULT_JS_HARD_STOP_TICKS_CL); -+ UPDATE_TIMEOUT(hard_stop_ticks_dumping, -+ js_hard_stop_ms_dumping, -+ DEFAULT_JS_HARD_STOP_TICKS_DUMPING); -+ UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? -+ DEFAULT_JS_RESET_TICKS_SS_8408 : -+ DEFAULT_JS_RESET_TICKS_SS); -+ UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, -+ DEFAULT_JS_RESET_TICKS_CL); -+ UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, -+ DEFAULT_JS_RESET_TICKS_DUMPING); -+ -+ kbase_js_set_timeouts(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return count; -+ } -+ -+ dev_err(kbdev->dev, "Couldn't process js_timeouts write operation.\n" -+ "Use format \n" -+ "Write 0 for no change, -1 to restore default timeout\n"); -+ return -EINVAL; -+} -+ -+static unsigned long get_js_timeout_in_ms( -+ u32 scheduling_period_ns, -+ u32 ticks) -+{ -+ u64 ms = (u64)ticks * scheduling_period_ns; -+ -+ do_div(ms, 1000000UL); -+ return ms; -+} -+ -+/** -+ * show_js_timeouts - Show callback for the js_timeouts sysfs file. -+ * -+ * This function is called to get the contents of the js_timeouts sysfs -+ * file. It returns the last set values written to the js_timeouts sysfs file. -+ * If the file didn't get written yet, the values will be current setting in -+ * use. -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_js_timeouts(struct device *dev, struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ unsigned long js_soft_stop_ms; -+ unsigned long js_soft_stop_ms_cl; -+ unsigned long js_hard_stop_ms_ss; -+ unsigned long js_hard_stop_ms_cl; -+ unsigned long js_hard_stop_ms_dumping; -+ unsigned long js_reset_ms_ss; -+ unsigned long js_reset_ms_cl; -+ unsigned long js_reset_ms_dumping; -+ u32 scheduling_period_ns; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ scheduling_period_ns = kbdev->js_data.scheduling_period_ns; -+ -+#define GET_TIMEOUT(name) get_js_timeout_in_ms(\ -+ scheduling_period_ns, \ -+ kbdev->js_data.name) -+ -+ js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); -+ js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); -+ js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); -+ js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); -+ js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); -+ js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); -+ js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); -+ js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); -+ -+#undef GET_TIMEOUT -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", -+ js_soft_stop_ms, js_soft_stop_ms_cl, -+ js_hard_stop_ms_ss, js_hard_stop_ms_cl, -+ js_hard_stop_ms_dumping, js_reset_ms_ss, -+ js_reset_ms_cl, js_reset_ms_dumping); -+ -+ if (ret >= PAGE_SIZE) { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/* -+ * The sysfs file js_timeouts. -+ * -+ * This is used to override the current job scheduler values for -+ * JS_STOP_STOP_TICKS_SS -+ * JS_STOP_STOP_TICKS_CL -+ * JS_HARD_STOP_TICKS_SS -+ * JS_HARD_STOP_TICKS_CL -+ * JS_HARD_STOP_TICKS_DUMPING -+ * JS_RESET_TICKS_SS -+ * JS_RESET_TICKS_CL -+ * JS_RESET_TICKS_DUMPING. -+ */ -+static DEVICE_ATTR(js_timeouts, S_IRUGO | S_IWUSR, show_js_timeouts, set_js_timeouts); -+ -+static u32 get_new_js_timeout( -+ u32 old_period, -+ u32 old_ticks, -+ u32 new_scheduling_period_ns) -+{ -+ u64 ticks = (u64)old_period * (u64)old_ticks; -+ do_div(ticks, new_scheduling_period_ns); -+ return ticks?ticks:1; -+} -+ -+/** -+ * set_js_scheduling_period - Store callback for the js_scheduling_period sysfs -+ * file -+ * @dev: The device the sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the js_scheduling_period sysfs file is written -+ * to. It checks the data written, and if valid updates the js_scheduling_period -+ * value -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_js_scheduling_period(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int ret; -+ unsigned int js_scheduling_period; -+ u32 new_scheduling_period_ns; -+ u32 old_period; -+ struct kbasep_js_device_data *js_data; -+ unsigned long flags; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ js_data = &kbdev->js_data; -+ -+ ret = kstrtouint(buf, 0, &js_scheduling_period); -+ if (ret || !js_scheduling_period) { -+ dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" -+ "Use format \n"); -+ return -EINVAL; -+ } -+ -+ new_scheduling_period_ns = js_scheduling_period * 1000000; -+ -+ /* Update scheduling timeouts */ -+ mutex_lock(&js_data->runpool_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* If no contexts have been scheduled since js_timeouts was last written -+ * to, the new timeouts might not have been latched yet. So check if an -+ * update is pending and use the new values if necessary. */ -+ -+ /* Use previous 'new' scheduling period as a base if present. */ -+ old_period = js_data->scheduling_period_ns; -+ -+#define SET_TIMEOUT(name) \ -+ (js_data->name = get_new_js_timeout(\ -+ old_period, \ -+ kbdev->js_data.name, \ -+ new_scheduling_period_ns)) -+ -+ SET_TIMEOUT(soft_stop_ticks); -+ SET_TIMEOUT(soft_stop_ticks_cl); -+ SET_TIMEOUT(hard_stop_ticks_ss); -+ SET_TIMEOUT(hard_stop_ticks_cl); -+ SET_TIMEOUT(hard_stop_ticks_dumping); -+ SET_TIMEOUT(gpu_reset_ticks_ss); -+ SET_TIMEOUT(gpu_reset_ticks_cl); -+ SET_TIMEOUT(gpu_reset_ticks_dumping); -+ -+#undef SET_TIMEOUT -+ -+ js_data->scheduling_period_ns = new_scheduling_period_ns; -+ -+ kbase_js_set_timeouts(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&js_data->runpool_mutex); -+ -+ dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", -+ js_scheduling_period); -+ -+ return count; -+} -+ -+/** -+ * show_js_scheduling_period - Show callback for the js_scheduling_period sysfs -+ * entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current period used for the JS scheduling -+ * period. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_js_scheduling_period(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ u32 period; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ period = kbdev->js_data.scheduling_period_ns; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%d\n", -+ period / 1000000); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR(js_scheduling_period, S_IRUGO | S_IWUSR, -+ show_js_scheduling_period, set_js_scheduling_period); -+ -+#if !MALI_CUSTOMER_RELEASE -+/** -+ * set_force_replay - Store callback for the force_replay sysfs file. -+ * -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_force_replay(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ if (!strncmp("limit=", buf, MIN(6, count))) { -+ int force_replay_limit; -+ int items = sscanf(buf, "limit=%u", &force_replay_limit); -+ -+ if (items == 1) { -+ kbdev->force_replay_random = false; -+ kbdev->force_replay_limit = force_replay_limit; -+ kbdev->force_replay_count = 0; -+ -+ return count; -+ } -+ } else if (!strncmp("random_limit", buf, MIN(12, count))) { -+ kbdev->force_replay_random = true; -+ kbdev->force_replay_count = 0; -+ -+ return count; -+ } else if (!strncmp("norandom_limit", buf, MIN(14, count))) { -+ kbdev->force_replay_random = false; -+ kbdev->force_replay_limit = KBASEP_FORCE_REPLAY_DISABLED; -+ kbdev->force_replay_count = 0; -+ -+ return count; -+ } else if (!strncmp("core_req=", buf, MIN(9, count))) { -+ unsigned int core_req; -+ int items = sscanf(buf, "core_req=%x", &core_req); -+ -+ if (items == 1) { -+ kbdev->force_replay_core_req = (base_jd_core_req)core_req; -+ -+ return count; -+ } -+ } -+ dev_err(kbdev->dev, "Couldn't process force_replay write operation.\nPossible settings: limit=, random_limit, norandom_limit, core_req=\n"); -+ return -EINVAL; -+} -+ -+/** -+ * show_force_replay - Show callback for the force_replay sysfs file. -+ * -+ * This function is called to get the contents of the force_replay sysfs -+ * file. It returns the last set value written to the force_replay sysfs file. -+ * If the file didn't get written yet, the values will be 0. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_force_replay(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ if (kbdev->force_replay_random) -+ ret = scnprintf(buf, PAGE_SIZE, -+ "limit=0\nrandom_limit\ncore_req=%x\n", -+ kbdev->force_replay_core_req); -+ else -+ ret = scnprintf(buf, PAGE_SIZE, -+ "limit=%u\nnorandom_limit\ncore_req=%x\n", -+ kbdev->force_replay_limit, -+ kbdev->force_replay_core_req); -+ -+ if (ret >= PAGE_SIZE) { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/* -+ * The sysfs file force_replay. -+ */ -+static DEVICE_ATTR(force_replay, S_IRUGO | S_IWUSR, show_force_replay, -+ set_force_replay); -+#endif /* !MALI_CUSTOMER_RELEASE */ -+ -+#ifdef CONFIG_MALI_DEBUG -+static ssize_t set_js_softstop_always(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int ret; -+ int softstop_always; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = kstrtoint(buf, 0, &softstop_always); -+ if (ret || ((softstop_always != 0) && (softstop_always != 1))) { -+ dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" -+ "Use format \n"); -+ return -EINVAL; -+ } -+ -+ kbdev->js_data.softstop_always = (bool) softstop_always; -+ dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", -+ (kbdev->js_data.softstop_always) ? -+ "Enabled" : "Disabled"); -+ return count; -+} -+ -+static ssize_t show_js_softstop_always(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); -+ -+ if (ret >= PAGE_SIZE) { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/* -+ * By default, soft-stops are disabled when only a single context is present. -+ * The ability to enable soft-stop when only a single context is present can be -+ * used for debug and unit-testing purposes. -+ * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) -+ */ -+static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); -+#endif /* CONFIG_MALI_DEBUG */ -+ -+#ifdef CONFIG_MALI_DEBUG -+typedef void (kbasep_debug_command_func) (struct kbase_device *); -+ -+enum kbasep_debug_command_code { -+ KBASEP_DEBUG_COMMAND_DUMPTRACE, -+ -+ /* This must be the last enum */ -+ KBASEP_DEBUG_COMMAND_COUNT -+}; -+ -+struct kbasep_debug_command { -+ char *str; -+ kbasep_debug_command_func *func; -+}; -+ -+/* Debug commands supported by the driver */ -+static const struct kbasep_debug_command debug_commands[] = { -+ { -+ .str = "dumptrace", -+ .func = &kbasep_trace_dump, -+ } -+}; -+ -+/** -+ * show_debug - Show callback for the debug_command sysfs file. -+ * -+ * This function is called to get the contents of the debug_command sysfs -+ * file. This is a list of the available debug commands, separated by newlines. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_debug(struct device *dev, struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ int i; -+ ssize_t ret = 0; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < PAGE_SIZE; i++) -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); -+ -+ if (ret >= PAGE_SIZE) { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/** -+ * issue_debug - Store callback for the debug_command sysfs file. -+ * -+ * This function is called when the debug_command sysfs file is written to. -+ * It matches the requested command against the available commands, and if -+ * a matching command is found calls the associated function from -+ * @debug_commands to issue the command. -+ * -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t issue_debug(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int i; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { -+ if (sysfs_streq(debug_commands[i].str, buf)) { -+ debug_commands[i].func(kbdev); -+ return count; -+ } -+ } -+ -+ /* Debug Command not found */ -+ dev_err(dev, "debug_command: command not known\n"); -+ return -EINVAL; -+} -+ -+/* The sysfs file debug_command. -+ * -+ * This is used to issue general debug commands to the device driver. -+ * Reading it will produce a list of debug commands, separated by newlines. -+ * Writing to it with one of those commands will issue said command. -+ */ -+static DEVICE_ATTR(debug_command, S_IRUGO | S_IWUSR, show_debug, issue_debug); -+#endif /* CONFIG_MALI_DEBUG */ -+ -+/** -+ * kbase_show_gpuinfo - Show callback for the gpuinfo sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get a description of the present Mali -+ * GPU via the gpuinfo sysfs entry. This includes the GPU family, the -+ * number of cores, the hardware version and the raw product id. For -+ * example -+ * -+ * Mali-T60x MP4 r0p0 0x6956 -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t kbase_show_gpuinfo(struct device *dev, -+ struct device_attribute *attr, char *buf) -+{ -+ static const struct gpu_product_id_name { -+ unsigned id; -+ char *name; -+ } gpu_product_id_names[] = { -+ { .id = GPU_ID_PI_T60X, .name = "Mali-T60x" }, -+ { .id = GPU_ID_PI_T62X, .name = "Mali-T62x" }, -+ { .id = GPU_ID_PI_T72X, .name = "Mali-T72x" }, -+ { .id = GPU_ID_PI_T76X, .name = "Mali-T76x" }, -+ { .id = GPU_ID_PI_T82X, .name = "Mali-T82x" }, -+ { .id = GPU_ID_PI_T83X, .name = "Mali-T83x" }, -+ { .id = GPU_ID_PI_T86X, .name = "Mali-T86x" }, -+ { .id = GPU_ID_PI_TFRX, .name = "Mali-T88x" }, -+ { .id = GPU_ID2_PRODUCT_TMIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -+ .name = "Mali-G71" }, -+ { .id = GPU_ID2_PRODUCT_THEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -+ .name = "Mali-THEx" }, -+ { .id = GPU_ID2_PRODUCT_TSIX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -+ .name = "Mali-G51" }, -+ }; -+ const char *product_name = "(Unknown Mali GPU)"; -+ struct kbase_device *kbdev; -+ u32 gpu_id; -+ unsigned product_id, product_id_mask; -+ unsigned i; -+ bool is_new_format; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ product_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ is_new_format = GPU_ID_IS_NEW_FORMAT(product_id); -+ product_id_mask = -+ (is_new_format ? -+ GPU_ID2_PRODUCT_MODEL : -+ GPU_ID_VERSION_PRODUCT_ID) >> -+ GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ -+ for (i = 0; i < ARRAY_SIZE(gpu_product_id_names); ++i) { -+ const struct gpu_product_id_name *p = &gpu_product_id_names[i]; -+ -+ if ((GPU_ID_IS_NEW_FORMAT(p->id) == is_new_format) && -+ (p->id & product_id_mask) == -+ (product_id & product_id_mask)) { -+ product_name = p->name; -+ break; -+ } -+ } -+ -+ return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%04X\n", -+ product_name, kbdev->gpu_props.num_cores, -+ (gpu_id & GPU_ID_VERSION_MAJOR) >> GPU_ID_VERSION_MAJOR_SHIFT, -+ (gpu_id & GPU_ID_VERSION_MINOR) >> GPU_ID_VERSION_MINOR_SHIFT, -+ product_id); -+} -+static DEVICE_ATTR(gpuinfo, S_IRUGO, kbase_show_gpuinfo, NULL); -+ -+/** -+ * set_dvfs_period - Store callback for the dvfs_period sysfs file. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the dvfs_period sysfs file is written to. It -+ * checks the data written, and if valid updates the DVFS period variable, -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_dvfs_period(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int ret; -+ int dvfs_period; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = kstrtoint(buf, 0, &dvfs_period); -+ if (ret || dvfs_period <= 0) { -+ dev_err(kbdev->dev, "Couldn't process dvfs_period write operation.\n" -+ "Use format \n"); -+ return -EINVAL; -+ } -+ -+ kbdev->pm.dvfs_period = dvfs_period; -+ dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); -+ -+ return count; -+} -+ -+/** -+ * show_dvfs_period - Show callback for the dvfs_period sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current period used for the DVFS sample -+ * timer. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_dvfs_period(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->pm.dvfs_period); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR(dvfs_period, S_IRUGO | S_IWUSR, show_dvfs_period, -+ set_dvfs_period); -+ -+/** -+ * set_pm_poweroff - Store callback for the pm_poweroff sysfs file. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the pm_poweroff sysfs file is written to. -+ * -+ * This file contains three values separated by whitespace. The values -+ * are gpu_poweroff_time (the period of the poweroff timer, in ns), -+ * poweroff_shader_ticks (the number of poweroff timer ticks before an idle -+ * shader is powered off), and poweroff_gpu_ticks (the number of poweroff timer -+ * ticks before the GPU is powered off), in that order. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_pm_poweroff(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int items; -+ s64 gpu_poweroff_time; -+ int poweroff_shader_ticks, poweroff_gpu_ticks; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ items = sscanf(buf, "%llu %u %u", &gpu_poweroff_time, -+ &poweroff_shader_ticks, -+ &poweroff_gpu_ticks); -+ if (items != 3) { -+ dev_err(kbdev->dev, "Couldn't process pm_poweroff write operation.\n" -+ "Use format \n"); -+ return -EINVAL; -+ } -+ -+ kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); -+ kbdev->pm.poweroff_shader_ticks = poweroff_shader_ticks; -+ kbdev->pm.poweroff_gpu_ticks = poweroff_gpu_ticks; -+ -+ return count; -+} -+ -+/** -+ * show_pm_poweroff - Show callback for the pm_poweroff sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current period used for the DVFS sample -+ * timer. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_pm_poweroff(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%llu %u %u\n", -+ ktime_to_ns(kbdev->pm.gpu_poweroff_time), -+ kbdev->pm.poweroff_shader_ticks, -+ kbdev->pm.poweroff_gpu_ticks); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, -+ set_pm_poweroff); -+ -+/** -+ * set_reset_timeout - Store callback for the reset_timeout sysfs file. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the reset_timeout sysfs file is written to. It -+ * checks the data written, and if valid updates the reset timeout. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_reset_timeout(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ int ret; -+ int reset_timeout; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = kstrtoint(buf, 0, &reset_timeout); -+ if (ret || reset_timeout <= 0) { -+ dev_err(kbdev->dev, "Couldn't process reset_timeout write operation.\n" -+ "Use format \n"); -+ return -EINVAL; -+ } -+ -+ kbdev->reset_timeout_ms = reset_timeout; -+ dev_dbg(kbdev->dev, "Reset timeout: %dms\n", reset_timeout); -+ -+ return count; -+} -+ -+/** -+ * show_reset_timeout - Show callback for the reset_timeout sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current reset timeout. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_reset_timeout(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->reset_timeout_ms); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR(reset_timeout, S_IRUGO | S_IWUSR, show_reset_timeout, -+ set_reset_timeout); -+ -+ -+ -+static ssize_t show_mem_pool_size(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%zu\n", -+ kbase_mem_pool_size(&kbdev->mem_pool)); -+ -+ return ret; -+} -+ -+static ssize_t set_mem_pool_size(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ size_t new_size; -+ int err; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ err = kstrtoul(buf, 0, (unsigned long *)&new_size); -+ if (err) -+ return err; -+ -+ kbase_mem_pool_trim(&kbdev->mem_pool, new_size); -+ -+ return count; -+} -+ -+static DEVICE_ATTR(mem_pool_size, S_IRUGO | S_IWUSR, show_mem_pool_size, -+ set_mem_pool_size); -+ -+static ssize_t show_mem_pool_max_size(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%zu\n", -+ kbase_mem_pool_max_size(&kbdev->mem_pool)); -+ -+ return ret; -+} -+ -+static ssize_t set_mem_pool_max_size(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ size_t new_max_size; -+ int err; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ err = kstrtoul(buf, 0, (unsigned long *)&new_max_size); -+ if (err) -+ return -EINVAL; -+ -+ kbase_mem_pool_set_max_size(&kbdev->mem_pool, new_max_size); -+ -+ return count; -+} -+ -+static DEVICE_ATTR(mem_pool_max_size, S_IRUGO | S_IWUSR, show_mem_pool_max_size, -+ set_mem_pool_max_size); -+ -+#ifdef CONFIG_DEBUG_FS -+ -+/* Number of entries in serialize_jobs_settings[] */ -+#define NR_SERIALIZE_JOBS_SETTINGS 5 -+/* Maximum string length in serialize_jobs_settings[].name */ -+#define MAX_SERIALIZE_JOBS_NAME_LEN 16 -+ -+static struct -+{ -+ char *name; -+ u8 setting; -+} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { -+ {"none", 0}, -+ {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, -+ {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, -+ {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, -+ {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | -+ KBASE_SERIALIZE_RESET} -+}; -+ -+/** -+ * kbasep_serialize_jobs_seq_show - Show callback for the serialize_jobs debugfs -+ * file -+ * @sfile: seq_file pointer -+ * @data: Private callback data -+ * -+ * This function is called to get the contents of the serialize_jobs debugfs -+ * file. This is a list of the available settings with the currently active one -+ * surrounded by square brackets. -+ * -+ * Return: 0 on success, or an error code on error -+ */ -+static int kbasep_serialize_jobs_seq_show(struct seq_file *sfile, void *data) -+{ -+ struct kbase_device *kbdev = sfile->private; -+ int i; -+ -+ CSTD_UNUSED(data); -+ -+ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { -+ if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) -+ seq_printf(sfile, "[%s] ", -+ serialize_jobs_settings[i].name); -+ else -+ seq_printf(sfile, "%s ", -+ serialize_jobs_settings[i].name); -+ } -+ -+ seq_puts(sfile, "\n"); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs -+ * debugfs file. -+ * @file: File pointer -+ * @ubuf: User buffer containing data to store -+ * @count: Number of bytes in user buffer -+ * @ppos: File position -+ * -+ * This function is called when the serialize_jobs debugfs file is written to. -+ * It matches the requested setting against the available settings and if a -+ * matching setting is found updates kbdev->serialize_jobs. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, -+ const char __user *ubuf, size_t count, loff_t *ppos) -+{ -+ struct seq_file *s = file->private_data; -+ struct kbase_device *kbdev = s->private; -+ char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; -+ int i; -+ bool valid = false; -+ -+ CSTD_UNUSED(ppos); -+ -+ count = min_t(size_t, sizeof(buf) - 1, count); -+ if (copy_from_user(buf, ubuf, count)) -+ return -EFAULT; -+ -+ buf[count] = 0; -+ -+ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { -+ if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { -+ kbdev->serialize_jobs = -+ serialize_jobs_settings[i].setting; -+ valid = true; -+ break; -+ } -+ } -+ -+ if (!valid) { -+ dev_err(kbdev->dev, "serialize_jobs: invalid setting\n"); -+ return -EINVAL; -+ } -+ -+ return count; -+} -+ -+/** -+ * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs -+ * debugfs file -+ * @in: inode pointer -+ * @file: file pointer -+ * -+ * Return: Zero on success, error code on failure -+ */ -+static int kbasep_serialize_jobs_debugfs_open(struct inode *in, -+ struct file *file) -+{ -+ return single_open(file, kbasep_serialize_jobs_seq_show, in->i_private); -+} -+ -+static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { -+ .open = kbasep_serialize_jobs_debugfs_open, -+ .read = seq_read, -+ .write = kbasep_serialize_jobs_debugfs_write, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+static int kbasep_protected_mode_init(struct kbase_device *kbdev) -+{ -+#ifdef CONFIG_OF -+ struct device_node *protected_node; -+ struct platform_device *pdev; -+ struct protected_mode_device *protected_dev; -+#endif -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) { -+ /* Use native protected ops */ -+ kbdev->protected_dev = kzalloc(sizeof(*kbdev->protected_dev), -+ GFP_KERNEL); -+ if (!kbdev->protected_dev) -+ return -ENOMEM; -+ kbdev->protected_dev->data = kbdev; -+ kbdev->protected_ops = &kbase_native_protected_ops; -+ kbdev->protected_mode_support = true; -+ return 0; -+ } -+ -+ kbdev->protected_mode_support = false; -+ -+#ifdef CONFIG_OF -+ protected_node = of_parse_phandle(kbdev->dev->of_node, -+ "protected-mode-switcher", 0); -+ -+ if (!protected_node) -+ protected_node = of_parse_phandle(kbdev->dev->of_node, -+ "secure-mode-switcher", 0); -+ -+ if (!protected_node) { -+ /* If protected_node cannot be looked up then we assume -+ * protected mode is not supported on this platform. */ -+ dev_info(kbdev->dev, "Protected mode not available\n"); -+ return 0; -+ } -+ -+ pdev = of_find_device_by_node(protected_node); -+ if (!pdev) -+ return -EINVAL; -+ -+ protected_dev = platform_get_drvdata(pdev); -+ if (!protected_dev) -+ return -EPROBE_DEFER; -+ -+ kbdev->protected_ops = &protected_dev->ops; -+ kbdev->protected_dev = protected_dev; -+ -+ if (kbdev->protected_ops) { -+ int err; -+ -+ /* Make sure protected mode is disabled on startup */ -+ mutex_lock(&kbdev->pm.lock); -+ err = kbdev->protected_ops->protected_mode_disable( -+ kbdev->protected_dev); -+ mutex_unlock(&kbdev->pm.lock); -+ -+ /* protected_mode_disable() returns -EINVAL if not supported */ -+ kbdev->protected_mode_support = (err != -EINVAL); -+ } -+#endif -+ return 0; -+} -+ -+static void kbasep_protected_mode_term(struct kbase_device *kbdev) -+{ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) -+ kfree(kbdev->protected_dev); -+} -+ -+#ifdef CONFIG_MALI_NO_MALI -+static int kbase_common_reg_map(struct kbase_device *kbdev) -+{ -+ return 0; -+} -+static void kbase_common_reg_unmap(struct kbase_device * const kbdev) -+{ -+} -+#else /* CONFIG_MALI_NO_MALI */ -+static int kbase_common_reg_map(struct kbase_device *kbdev) -+{ -+ int err = -ENOMEM; -+ -+ if (!request_mem_region(kbdev->reg_start, kbdev->reg_size, dev_name(kbdev->dev))) { -+ dev_err(kbdev->dev, "Register window unavailable\n"); -+ err = -EIO; -+ goto out_region; -+ } -+ -+ kbdev->reg = ioremap(kbdev->reg_start, kbdev->reg_size); -+ if (!kbdev->reg) { -+ dev_err(kbdev->dev, "Can't remap register window\n"); -+ err = -EINVAL; -+ goto out_ioremap; -+ } -+ -+ return 0; -+ -+ out_ioremap: -+ release_mem_region(kbdev->reg_start, kbdev->reg_size); -+ out_region: -+ return err; -+} -+ -+static void kbase_common_reg_unmap(struct kbase_device * const kbdev) -+{ -+ if (kbdev->reg) { -+ iounmap(kbdev->reg); -+ release_mem_region(kbdev->reg_start, kbdev->reg_size); -+ kbdev->reg = NULL; -+ kbdev->reg_start = 0; -+ kbdev->reg_size = 0; -+ } -+} -+#endif /* CONFIG_MALI_NO_MALI */ -+ -+static int registers_map(struct kbase_device * const kbdev) -+{ -+ -+ /* the first memory resource is the physical address of the GPU -+ * registers */ -+ struct platform_device *pdev = to_platform_device(kbdev->dev); -+ struct resource *reg_res; -+ int err; -+ -+ reg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); -+ if (!reg_res) { -+ dev_err(kbdev->dev, "Invalid register resource\n"); -+ return -ENOENT; -+ } -+ -+ kbdev->reg_start = reg_res->start; -+ kbdev->reg_size = resource_size(reg_res); -+ -+ err = kbase_common_reg_map(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Failed to map registers\n"); -+ return err; -+ } -+ -+ return 0; -+} -+ -+static void registers_unmap(struct kbase_device *kbdev) -+{ -+ kbase_common_reg_unmap(kbdev); -+} -+ -+static int power_control_init(struct platform_device *pdev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); -+ int err = 0; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_REGULATOR) -+ kbdev->regulator = regulator_get_optional(kbdev->dev, "mali"); -+ if (IS_ERR_OR_NULL(kbdev->regulator)) { -+ err = PTR_ERR(kbdev->regulator); -+ kbdev->regulator = NULL; -+ if (err == -EPROBE_DEFER) { -+ dev_err(&pdev->dev, "Failed to get regulator\n"); -+ return err; -+ } -+ dev_info(kbdev->dev, -+ "Continuing without Mali regulator control\n"); -+ /* Allow probe to continue without regulator */ -+ } -+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ -+ -+ kbdev->clock = clk_get(kbdev->dev, "clk_mali"); -+ if (IS_ERR_OR_NULL(kbdev->clock)) { -+ err = PTR_ERR(kbdev->clock); -+ kbdev->clock = NULL; -+ if (err == -EPROBE_DEFER) { -+ dev_err(&pdev->dev, "Failed to get clock\n"); -+ goto fail; -+ } -+ dev_info(kbdev->dev, "Continuing without Mali clock control\n"); -+ /* Allow probe to continue without clock. */ -+ } else { -+ err = clk_prepare(kbdev->clock); -+ if (err) { -+ dev_err(kbdev->dev, -+ "Failed to prepare and enable clock (%d)\n", -+ err); -+ goto fail; -+ } -+ } -+ -+ err = kbase_platform_rk_init_opp_table(kbdev); -+ if (err) -+ dev_err(kbdev->dev, "Failed to init_opp_table (%d)\n", err); -+ -+ return 0; -+ -+fail: -+ -+if (kbdev->clock != NULL) { -+ clk_put(kbdev->clock); -+ kbdev->clock = NULL; -+} -+ -+#ifdef CONFIG_REGULATOR -+ if (NULL != kbdev->regulator) { -+ regulator_put(kbdev->regulator); -+ kbdev->regulator = NULL; -+ } -+#endif -+ -+ return err; -+} -+ -+static void power_control_term(struct kbase_device *kbdev) -+{ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) || \ -+ defined(LSK_OPPV2_BACKPORT) -+ dev_pm_opp_of_remove_table(kbdev->dev); -+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) -+ of_free_opp_table(kbdev->dev); -+#endif -+ -+ if (kbdev->clock) { -+ clk_unprepare(kbdev->clock); -+ clk_put(kbdev->clock); -+ kbdev->clock = NULL; -+ } -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ -+ && defined(CONFIG_REGULATOR) -+ if (kbdev->regulator) { -+ regulator_put(kbdev->regulator); -+ kbdev->regulator = NULL; -+ } -+#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ -+} -+ -+#ifdef CONFIG_DEBUG_FS -+ -+#if KBASE_GPU_RESET_EN -+#include -+ -+static void trigger_quirks_reload(struct kbase_device *kbdev) -+{ -+ kbase_pm_context_active(kbdev); -+ if (kbase_prepare_to_reset_gpu(kbdev)) -+ kbase_reset_gpu(kbdev); -+ kbase_pm_context_idle(kbdev); -+} -+ -+#define MAKE_QUIRK_ACCESSORS(type) \ -+static int type##_quirks_set(void *data, u64 val) \ -+{ \ -+ struct kbase_device *kbdev; \ -+ kbdev = (struct kbase_device *)data; \ -+ kbdev->hw_quirks_##type = (u32)val; \ -+ trigger_quirks_reload(kbdev); \ -+ return 0;\ -+} \ -+\ -+static int type##_quirks_get(void *data, u64 *val) \ -+{ \ -+ struct kbase_device *kbdev;\ -+ kbdev = (struct kbase_device *)data;\ -+ *val = kbdev->hw_quirks_##type;\ -+ return 0;\ -+} \ -+DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ -+ type##_quirks_set, "%llu\n") -+ -+MAKE_QUIRK_ACCESSORS(sc); -+MAKE_QUIRK_ACCESSORS(tiler); -+MAKE_QUIRK_ACCESSORS(mmu); -+MAKE_QUIRK_ACCESSORS(jm); -+ -+#endif /* KBASE_GPU_RESET_EN */ -+ -+/** -+ * debugfs_protected_debug_mode_read - "protected_debug_mode" debugfs read -+ * @file: File object to read is for -+ * @buf: User buffer to populate with data -+ * @len: Length of user buffer -+ * @ppos: Offset within file object -+ * -+ * Retrieves the current status of protected debug mode -+ * (0 = disabled, 1 = enabled) -+ * -+ * Return: Number of bytes added to user buffer -+ */ -+static ssize_t debugfs_protected_debug_mode_read(struct file *file, -+ char __user *buf, size_t len, loff_t *ppos) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)file->private_data; -+ u32 gpu_status; -+ ssize_t ret_val; -+ -+ kbase_pm_context_active(kbdev); -+ gpu_status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL); -+ kbase_pm_context_idle(kbdev); -+ -+ if (gpu_status & GPU_DBGEN) -+ ret_val = simple_read_from_buffer(buf, len, ppos, "1\n", 2); -+ else -+ ret_val = simple_read_from_buffer(buf, len, ppos, "0\n", 2); -+ -+ return ret_val; -+} -+ -+/* -+ * struct fops_protected_debug_mode - "protected_debug_mode" debugfs fops -+ * -+ * Contains the file operations for the "protected_debug_mode" debugfs file -+ */ -+static const struct file_operations fops_protected_debug_mode = { -+ .open = simple_open, -+ .read = debugfs_protected_debug_mode_read, -+ .llseek = default_llseek, -+}; -+ -+static int kbase_device_debugfs_init(struct kbase_device *kbdev) -+{ -+ struct dentry *debugfs_ctx_defaults_directory; -+ int err; -+ -+ kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, -+ NULL); -+ if (!kbdev->mali_debugfs_directory) { -+ dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ kbdev->debugfs_ctx_directory = debugfs_create_dir("ctx", -+ kbdev->mali_debugfs_directory); -+ if (!kbdev->debugfs_ctx_directory) { -+ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx directory\n"); -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", -+ kbdev->debugfs_ctx_directory); -+ if (!debugfs_ctx_defaults_directory) { -+ dev_err(kbdev->dev, "Couldn't create mali debugfs ctx defaults directory\n"); -+ err = -ENOMEM; -+ goto out; -+ } -+ -+#if !MALI_CUSTOMER_RELEASE -+ kbasep_regs_dump_debugfs_init(kbdev); -+#endif /* !MALI_CUSTOMER_RELEASE */ -+ kbasep_regs_history_debugfs_init(kbdev); -+ -+ kbase_debug_job_fault_debugfs_init(kbdev); -+ kbasep_gpu_memory_debugfs_init(kbdev); -+ kbase_as_fault_debugfs_init(kbdev); -+#if KBASE_GPU_RESET_EN -+ /* fops_* variables created by invocations of macro -+ * MAKE_QUIRK_ACCESSORS() above. */ -+ debugfs_create_file("quirks_sc", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &fops_sc_quirks); -+ debugfs_create_file("quirks_tiler", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &fops_tiler_quirks); -+ debugfs_create_file("quirks_mmu", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &fops_mmu_quirks); -+ debugfs_create_file("quirks_jm", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &fops_jm_quirks); -+#endif /* KBASE_GPU_RESET_EN */ -+ -+#ifndef CONFIG_MALI_COH_USER -+ debugfs_create_bool("infinite_cache", 0644, -+ debugfs_ctx_defaults_directory, -+ (bool*)&(kbdev->infinite_cache_active_default)); -+#endif /* CONFIG_MALI_COH_USER */ -+ -+ debugfs_create_size_t("mem_pool_max_size", 0644, -+ debugfs_ctx_defaults_directory, -+ &kbdev->mem_pool_max_size_default); -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { -+ debugfs_create_file("protected_debug_mode", S_IRUGO, -+ kbdev->mali_debugfs_directory, kbdev, -+ &fops_protected_debug_mode); -+ } -+ -+#if KBASE_TRACE_ENABLE -+ kbasep_trace_debugfs_init(kbdev); -+#endif /* KBASE_TRACE_ENABLE */ -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ kbasep_trace_timeline_debugfs_init(kbdev); -+#endif /* CONFIG_MALI_TRACE_TIMELINE */ -+ -+#ifdef CONFIG_MALI_DEVFREQ -+#ifdef CONFIG_DEVFREQ_THERMAL -+ if (kbdev->inited_subsys & inited_devfreq) -+ kbase_ipa_debugfs_init(kbdev); -+#endif /* CONFIG_DEVFREQ_THERMAL */ -+#endif /* CONFIG_MALI_DEVFREQ */ -+ -+#ifdef CONFIG_DEBUG_FS -+ debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_serialize_jobs_debugfs_fops); -+#endif /* CONFIG_DEBUG_FS */ -+ -+ return 0; -+ -+out: -+ debugfs_remove_recursive(kbdev->mali_debugfs_directory); -+ return err; -+} -+ -+static void kbase_device_debugfs_term(struct kbase_device *kbdev) -+{ -+ debugfs_remove_recursive(kbdev->mali_debugfs_directory); -+} -+ -+#else /* CONFIG_DEBUG_FS */ -+static inline int kbase_device_debugfs_init(struct kbase_device *kbdev) -+{ -+ return 0; -+} -+ -+static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { } -+#endif /* CONFIG_DEBUG_FS */ -+ -+static void kbase_device_coherency_init(struct kbase_device *kbdev, -+ unsigned prod_id) -+{ -+#ifdef CONFIG_OF -+ u32 supported_coherency_bitmap = -+ kbdev->gpu_props.props.raw_props.coherency_mode; -+ const void *coherency_override_dts; -+ u32 override_coherency; -+ -+ /* Only for tMIx : -+ * (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly -+ * documented for tMIx so force correct value here. -+ */ -+ if (GPU_ID_IS_NEW_FORMAT(prod_id) && -+ (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == -+ GPU_ID2_PRODUCT_TMIX)) -+ if (supported_coherency_bitmap == -+ COHERENCY_FEATURE_BIT(COHERENCY_ACE)) -+ supported_coherency_bitmap |= -+ COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE); -+ -+#endif /* CONFIG_OF */ -+ -+ kbdev->system_coherency = COHERENCY_NONE; -+ -+ /* device tree may override the coherency */ -+#ifdef CONFIG_OF -+ coherency_override_dts = of_get_property(kbdev->dev->of_node, -+ "system-coherency", -+ NULL); -+ if (coherency_override_dts) { -+ -+ override_coherency = be32_to_cpup(coherency_override_dts); -+ -+ if ((override_coherency <= COHERENCY_NONE) && -+ (supported_coherency_bitmap & -+ COHERENCY_FEATURE_BIT(override_coherency))) { -+ -+ kbdev->system_coherency = override_coherency; -+ -+ dev_info(kbdev->dev, -+ "Using coherency mode %u set from dtb", -+ override_coherency); -+ } else -+ dev_warn(kbdev->dev, -+ "Ignoring unsupported coherency mode %u set from dtb", -+ override_coherency); -+ } -+ -+#endif /* CONFIG_OF */ -+ -+ kbdev->gpu_props.props.raw_props.coherency_mode = -+ kbdev->system_coherency; -+} -+ -+#ifdef CONFIG_MALI_FPGA_BUS_LOGGER -+ -+/* Callback used by the kbase bus logger client, to initiate a GPU reset -+ * when the bus log is restarted. GPU reset is used as reference point -+ * in HW bus log analyses. -+ */ -+static void kbase_logging_started_cb(void *data) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)data; -+ -+ if (kbase_prepare_to_reset_gpu(kbdev)) -+ kbase_reset_gpu(kbdev); -+ dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); -+} -+#endif -+ -+static struct attribute *kbase_attrs[] = { -+#ifdef CONFIG_MALI_DEBUG -+ &dev_attr_debug_command.attr, -+ &dev_attr_js_softstop_always.attr, -+#endif -+#if !MALI_CUSTOMER_RELEASE -+ &dev_attr_force_replay.attr, -+#endif -+ &dev_attr_js_timeouts.attr, -+ &dev_attr_soft_job_timeout.attr, -+ &dev_attr_gpuinfo.attr, -+ &dev_attr_dvfs_period.attr, -+ &dev_attr_pm_poweroff.attr, -+ &dev_attr_reset_timeout.attr, -+ &dev_attr_js_scheduling_period.attr, -+ &dev_attr_power_policy.attr, -+ &dev_attr_core_availability_policy.attr, -+ &dev_attr_core_mask.attr, -+ &dev_attr_mem_pool_size.attr, -+ &dev_attr_mem_pool_max_size.attr, -+ NULL -+}; -+ -+static const struct attribute_group kbase_attr_group = { -+ .attrs = kbase_attrs, -+}; -+ -+static int kbase_platform_device_remove(struct platform_device *pdev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); -+ const struct list_head *dev_list; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ kfree(kbdev->gpu_props.prop_buffer); -+ -+#ifdef CONFIG_MALI_FPGA_BUS_LOGGER -+ if (kbdev->inited_subsys & inited_buslogger) { -+ bl_core_client_unregister(kbdev->buslogger); -+ kbdev->inited_subsys &= ~inited_buslogger; -+ } -+#endif -+ -+ -+ if (kbdev->inited_subsys & inited_dev_list) { -+ dev_list = kbase_dev_list_get(); -+ list_del(&kbdev->entry); -+ kbase_dev_list_put(dev_list); -+ kbdev->inited_subsys &= ~inited_dev_list; -+ } -+ -+ if (kbdev->inited_subsys & inited_misc_register) { -+ misc_deregister(&kbdev->mdev); -+ kbdev->inited_subsys &= ~inited_misc_register; -+ } -+ -+ if (kbdev->inited_subsys & inited_sysfs_group) { -+ sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); -+ kbdev->inited_subsys &= ~inited_sysfs_group; -+ } -+ -+ if (kbdev->inited_subsys & inited_get_device) { -+ put_device(kbdev->dev); -+ kbdev->inited_subsys &= ~inited_get_device; -+ } -+ -+ if (kbdev->inited_subsys & inited_debugfs) { -+ kbase_device_debugfs_term(kbdev); -+ kbdev->inited_subsys &= ~inited_debugfs; -+ } -+ -+ if (kbdev->inited_subsys & inited_job_fault) { -+ kbase_debug_job_fault_dev_term(kbdev); -+ kbdev->inited_subsys &= ~inited_job_fault; -+ } -+ if (kbdev->inited_subsys & inited_vinstr) { -+ kbase_vinstr_term(kbdev->vinstr_ctx); -+ kbdev->inited_subsys &= ~inited_vinstr; -+ } -+ -+#ifdef CONFIG_MALI_DEVFREQ -+ if (kbdev->inited_subsys & inited_devfreq) { -+ kbase_devfreq_term(kbdev); -+ kbdev->inited_subsys &= ~inited_devfreq; -+ } -+#endif -+ -+ if (kbdev->inited_subsys & inited_backend_late) { -+ kbase_backend_late_term(kbdev); -+ kbdev->inited_subsys &= ~inited_backend_late; -+ } -+ -+ if (kbdev->inited_subsys & inited_tlstream) { -+ kbase_tlstream_term(); -+ kbdev->inited_subsys &= ~inited_tlstream; -+ } -+ -+ /* Bring job and mem sys to a halt before we continue termination */ -+ -+ if (kbdev->inited_subsys & inited_js) -+ kbasep_js_devdata_halt(kbdev); -+ -+ if (kbdev->inited_subsys & inited_mem) -+ kbase_mem_halt(kbdev); -+ -+ if (kbdev->inited_subsys & inited_protected) { -+ kbasep_protected_mode_term(kbdev); -+ kbdev->inited_subsys &= ~inited_protected; -+ } -+ -+ if (kbdev->inited_subsys & inited_js) { -+ kbasep_js_devdata_term(kbdev); -+ kbdev->inited_subsys &= ~inited_js; -+ } -+ -+ if (kbdev->inited_subsys & inited_mem) { -+ kbase_mem_term(kbdev); -+ kbdev->inited_subsys &= ~inited_mem; -+ } -+ -+ if (kbdev->inited_subsys & inited_pm_runtime_init) { -+ kbdev->pm.callback_power_runtime_term(kbdev); -+ kbdev->inited_subsys &= ~inited_pm_runtime_init; -+ } -+ -+ if (kbdev->inited_subsys & inited_ctx_sched) { -+ kbase_ctx_sched_term(kbdev); -+ kbdev->inited_subsys &= ~inited_ctx_sched; -+ } -+ -+ if (kbdev->inited_subsys & inited_device) { -+ kbase_device_term(kbdev); -+ kbdev->inited_subsys &= ~inited_device; -+ } -+ -+ if (kbdev->inited_subsys & inited_backend_early) { -+ kbase_backend_early_term(kbdev); -+ kbdev->inited_subsys &= ~inited_backend_early; -+ } -+ -+ if (kbdev->inited_subsys & inited_io_history) { -+ kbase_io_history_term(&kbdev->io_history); -+ kbdev->inited_subsys &= ~inited_io_history; -+ } -+ -+ if (kbdev->inited_subsys & inited_power_control) { -+ power_control_term(kbdev); -+ kbdev->inited_subsys &= ~inited_power_control; -+ } -+ -+ if (kbdev->inited_subsys & inited_registers_map) { -+ registers_unmap(kbdev); -+ kbdev->inited_subsys &= ~inited_registers_map; -+ } -+ -+#ifdef CONFIG_MALI_NO_MALI -+ if (kbdev->inited_subsys & inited_gpu_device) { -+ gpu_device_destroy(kbdev); -+ kbdev->inited_subsys &= ~inited_gpu_device; -+ } -+#endif /* CONFIG_MALI_NO_MALI */ -+ -+ if (kbdev->inited_subsys != 0) -+ dev_err(kbdev->dev, "Missing sub system termination\n"); -+ -+ kbase_device_free(kbdev); -+ -+ return 0; -+} -+ -+extern void kbase_platform_rk_shutdown(struct kbase_device *kbdev); -+static void kbase_platform_device_shutdown(struct platform_device *pdev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(&pdev->dev); -+ -+ kbase_platform_rk_shutdown(kbdev); -+} -+ -+/* Number of register accesses for the buffer that we allocate during -+ * initialization time. The buffer size can be changed later via debugfs. */ -+#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512) -+ -+static int kbase_platform_device_probe(struct platform_device *pdev) -+{ -+ struct kbase_device *kbdev; -+ struct mali_base_gpu_core_props *core_props; -+ u32 gpu_id; -+ unsigned prod_id; -+ const struct list_head *dev_list; -+ int err = 0; -+ -+#ifdef CONFIG_OF -+ err = kbase_platform_early_init(); -+ if (err) { -+ dev_err(&pdev->dev, "Early platform initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+#endif -+ kbdev = kbase_device_alloc(); -+ if (!kbdev) { -+ dev_err(&pdev->dev, "Allocate device failed\n"); -+ kbase_platform_device_remove(pdev); -+ return -ENOMEM; -+ } -+ -+ kbdev->dev = &pdev->dev; -+ dev_set_drvdata(kbdev->dev, kbdev); -+ -+#ifdef CONFIG_MALI_NO_MALI -+ err = gpu_device_create(kbdev); -+ if (err) { -+ dev_err(&pdev->dev, "Dummy model initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_gpu_device; -+#endif /* CONFIG_MALI_NO_MALI */ -+ -+ err = assign_irqs(pdev); -+ if (err) { -+ dev_err(&pdev->dev, "IRQ search failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ -+ err = registers_map(kbdev); -+ if (err) { -+ dev_err(&pdev->dev, "Register map failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_registers_map; -+ -+ err = power_control_init(pdev); -+ if (err) { -+ dev_err(&pdev->dev, "Power control initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_power_control; -+ -+ err = kbase_io_history_init(&kbdev->io_history, -+ KBASEP_DEFAULT_REGISTER_HISTORY_SIZE); -+ if (err) { -+ dev_err(&pdev->dev, "Register access history initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return -ENOMEM; -+ } -+ kbdev->inited_subsys |= inited_io_history; -+ -+ err = kbase_backend_early_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Early backend initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_backend_early; -+ -+ scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name, -+ kbase_dev_nr); -+ -+ kbase_disjoint_init(kbdev); -+ -+ /* obtain min/max configured gpu frequencies */ -+ core_props = &(kbdev->gpu_props.props.core_props); -+ core_props->gpu_freq_khz_min = GPU_FREQ_KHZ_MIN; -+ core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX; -+ -+ err = kbase_device_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Device initialization failed (%d)\n", err); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_device; -+ -+ err = kbase_ctx_sched_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Context scheduler initialization failed (%d)\n", -+ err); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_ctx_sched; -+ -+ if (kbdev->pm.callback_power_runtime_init) { -+ err = kbdev->pm.callback_power_runtime_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, -+ "Runtime PM initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_pm_runtime_init; -+ } -+ -+ err = kbase_mem_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Memory subsystem initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_mem; -+ -+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ gpu_id &= GPU_ID_VERSION_PRODUCT_ID; -+ prod_id = gpu_id >> GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ -+ kbase_device_coherency_init(kbdev, prod_id); -+ -+ err = kbasep_protected_mode_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Protected mode subsystem initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_protected; -+ -+ dev_list = kbase_dev_list_get(); -+ list_add(&kbdev->entry, &kbase_dev_list); -+ kbase_dev_list_put(dev_list); -+ kbdev->inited_subsys |= inited_dev_list; -+ -+ err = kbasep_js_devdata_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Job JS devdata initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_js; -+ -+ err = kbase_tlstream_init(); -+ if (err) { -+ dev_err(kbdev->dev, "Timeline stream initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_tlstream; -+ -+ err = kbase_backend_late_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Late backend initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_backend_late; -+ -+#ifdef CONFIG_MALI_DEVFREQ -+ err = kbase_devfreq_init(kbdev); -+ if (!err) -+ kbdev->inited_subsys |= inited_devfreq; -+ else -+ dev_err(kbdev->dev, "Continuing without devfreq\n"); -+#endif /* CONFIG_MALI_DEVFREQ */ -+ -+ kbdev->vinstr_ctx = kbase_vinstr_init(kbdev); -+ if (!kbdev->vinstr_ctx) { -+ dev_err(kbdev->dev, -+ "Virtual instrumentation initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return -EINVAL; -+ } -+ kbdev->inited_subsys |= inited_vinstr; -+ -+ err = kbase_debug_job_fault_dev_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "Job fault debug initialization failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_job_fault; -+ -+ err = kbase_device_debugfs_init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "DebugFS initialization failed"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_debugfs; -+ -+ /* initialize the kctx list */ -+ mutex_init(&kbdev->kctx_list_lock); -+ INIT_LIST_HEAD(&kbdev->kctx_list); -+ -+ kbdev->mdev.minor = MISC_DYNAMIC_MINOR; -+ kbdev->mdev.name = kbdev->devname; -+ kbdev->mdev.fops = &kbase_fops; -+ kbdev->mdev.parent = get_device(kbdev->dev); -+ kbdev->inited_subsys |= inited_get_device; -+ -+ /* This needs to happen before registering the device with misc_register(), -+ * otherwise it causes a race condition between registering the device and a -+ * uevent event being generated for userspace, causing udev rules to run -+ * which might expect certain sysfs attributes present. As a result of the -+ * race condition we avoid, some Mali sysfs entries may have appeared to -+ * udev to not exist. -+ -+ * For more information, see -+ * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the -+ * paragraph that starts with "Word of warning", currently the second-last -+ * paragraph. -+ */ -+ err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); -+ if (err) { -+ dev_err(&pdev->dev, "SysFS group creation failed\n"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_sysfs_group; -+ -+ err = misc_register(&kbdev->mdev); -+ if (err) { -+ dev_err(kbdev->dev, "Misc device registration failed for %s\n", -+ kbdev->devname); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ kbdev->inited_subsys |= inited_misc_register; -+ -+ -+#ifdef CONFIG_MALI_FPGA_BUS_LOGGER -+ err = bl_core_client_register(kbdev->devname, -+ kbase_logging_started_cb, -+ kbdev, &kbdev->buslogger, -+ THIS_MODULE, NULL); -+ if (err == 0) { -+ kbdev->inited_subsys |= inited_buslogger; -+ bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); -+ } else { -+ dev_warn(kbdev->dev, "Bus log client registration failed\n"); -+ err = 0; -+ } -+#endif -+ -+ err = kbase_gpuprops_populate_user_buffer(kbdev); -+ if (err) { -+ dev_err(&pdev->dev, "GPU property population failed"); -+ kbase_platform_device_remove(pdev); -+ return err; -+ } -+ -+ dev_info(kbdev->dev, -+ "Probed as %s\n", dev_name(kbdev->mdev.this_device)); -+ -+ kbase_dev_nr++; -+ -+ return err; -+} -+ -+#undef KBASEP_DEFAULT_REGISTER_HISTORY_SIZE -+ -+/** -+ * kbase_device_suspend - Suspend callback from the OS. -+ * -+ * This is called by Linux when the device should suspend. -+ * -+ * @dev: The device to suspend -+ * -+ * Return: A standard Linux error code -+ */ -+static int kbase_device_suspend(struct device *dev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ if (kbdev->inited_subsys & inited_devfreq) -+ devfreq_suspend_device(kbdev->devfreq); -+#endif -+ -+ kbase_pm_suspend(kbdev); -+ return 0; -+} -+ -+/** -+ * kbase_device_resume - Resume callback from the OS. -+ * -+ * This is called by Linux when the device should resume from suspension. -+ * -+ * @dev: The device to resume -+ * -+ * Return: A standard Linux error code -+ */ -+static int kbase_device_resume(struct device *dev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ kbase_pm_resume(kbdev); -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ if (kbdev->inited_subsys & inited_devfreq) -+ devfreq_resume_device(kbdev->devfreq); -+#endif -+ return 0; -+} -+ -+/** -+ * kbase_device_runtime_suspend - Runtime suspend callback from the OS. -+ * -+ * This is called by Linux when the device should prepare for a condition in -+ * which it will not be able to communicate with the CPU(s) and RAM due to -+ * power management. -+ * -+ * @dev: The device to suspend -+ * -+ * Return: A standard Linux error code -+ */ -+#ifdef KBASE_PM_RUNTIME -+static int kbase_device_runtime_suspend(struct device *dev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ if (kbdev->inited_subsys & inited_devfreq) -+ devfreq_suspend_device(kbdev->devfreq); -+#endif -+ -+ if (kbdev->pm.backend.callback_power_runtime_off) { -+ kbdev->pm.backend.callback_power_runtime_off(kbdev); -+ dev_dbg(dev, "runtime suspend\n"); -+ } -+ return 0; -+} -+#endif /* KBASE_PM_RUNTIME */ -+ -+/** -+ * kbase_device_runtime_resume - Runtime resume callback from the OS. -+ * -+ * This is called by Linux when the device should go into a fully active state. -+ * -+ * @dev: The device to suspend -+ * -+ * Return: A standard Linux error code -+ */ -+ -+#ifdef KBASE_PM_RUNTIME -+static int kbase_device_runtime_resume(struct device *dev) -+{ -+ int ret = 0; -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ if (kbdev->pm.backend.callback_power_runtime_on) { -+ ret = kbdev->pm.backend.callback_power_runtime_on(kbdev); -+ dev_dbg(dev, "runtime resume\n"); -+ } -+ -+#if defined(CONFIG_MALI_DEVFREQ) && \ -+ (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ if (kbdev->inited_subsys & inited_devfreq) -+ devfreq_resume_device(kbdev->devfreq); -+#endif -+ -+ return ret; -+} -+#endif /* KBASE_PM_RUNTIME */ -+ -+ -+#ifdef KBASE_PM_RUNTIME -+/** -+ * kbase_device_runtime_idle - Runtime idle callback from the OS. -+ * @dev: The device to suspend -+ * -+ * This is called by Linux when the device appears to be inactive and it might -+ * be placed into a low power state. -+ * -+ * Return: 0 if device can be suspended, non-zero to avoid runtime autosuspend, -+ * otherwise a standard Linux error code -+ */ -+static int kbase_device_runtime_idle(struct device *dev) -+{ -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ /* Use platform specific implementation if it exists. */ -+ if (kbdev->pm.backend.callback_power_runtime_idle) -+ return kbdev->pm.backend.callback_power_runtime_idle(kbdev); -+ -+ return 0; -+} -+#endif /* KBASE_PM_RUNTIME */ -+ -+/* The power management operations for the platform driver. -+ */ -+static const struct dev_pm_ops kbase_pm_ops = { -+ .suspend = kbase_device_suspend, -+ .resume = kbase_device_resume, -+#ifdef KBASE_PM_RUNTIME -+ .runtime_suspend = kbase_device_runtime_suspend, -+ .runtime_resume = kbase_device_runtime_resume, -+ .runtime_idle = kbase_device_runtime_idle, -+#endif /* KBASE_PM_RUNTIME */ -+}; -+ -+#ifdef CONFIG_OF -+static const struct of_device_id kbase_dt_ids[] = { -+ { .compatible = "arm,malit7xx" }, -+ { .compatible = "arm,mali-midgard" }, -+ { /* sentinel */ } -+}; -+MODULE_DEVICE_TABLE(of, kbase_dt_ids); -+#endif -+ -+static struct platform_driver kbase_platform_driver = { -+ .probe = kbase_platform_device_probe, -+ .remove = kbase_platform_device_remove, -+ .shutdown = kbase_platform_device_shutdown, -+ .driver = { -+ .name = "midgard", -+ .owner = THIS_MODULE, -+ .pm = &kbase_pm_ops, -+ .of_match_table = of_match_ptr(kbase_dt_ids), -+ }, -+}; -+ -+/* -+ * The driver will not provide a shortcut to create the Mali platform device -+ * anymore when using Device Tree. -+ */ -+#ifdef CONFIG_OF -+module_platform_driver(kbase_platform_driver); -+#else -+ -+static int __init rockchip_gpu_init_driver(void) -+{ -+ return platform_driver_register(&kbase_platform_driver); -+} -+late_initcall(rockchip_gpu_init_driver); -+ -+static int __init kbase_driver_init(void) -+{ -+ int ret; -+ -+ ret = kbase_platform_early_init(); -+ if (ret) -+ return ret; -+ -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+ ret = kbase_platform_fake_register(); -+ if (ret) -+ return ret; -+#endif -+ ret = platform_driver_register(&kbase_platform_driver); -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+ if (ret) -+ kbase_platform_fake_unregister(); -+#endif -+ return ret; -+} -+ -+static void __exit kbase_driver_exit(void) -+{ -+ platform_driver_unregister(&kbase_platform_driver); -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+ kbase_platform_fake_unregister(); -+#endif -+} -+ -+module_init(kbase_driver_init); -+module_exit(kbase_driver_exit); -+ -+#endif /* CONFIG_OF */ -+ -+MODULE_LICENSE("GPL"); -+MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ -+ __stringify(BASE_UK_VERSION_MAJOR) "." \ -+ __stringify(BASE_UK_VERSION_MINOR) ")"); -+ -+#if defined(CONFIG_MALI_GATOR_SUPPORT) || defined(CONFIG_MALI_SYSTEM_TRACE) -+#define CREATE_TRACE_POINTS -+#endif -+ -+#ifdef CONFIG_MALI_GATOR_SUPPORT -+/* Create the trace points (otherwise we just get code to call a tracepoint) */ -+#include "mali_linux_trace.h" -+ -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_job_slots_event); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_status); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_on); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_pm_power_off); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_page_fault_insert_pages); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_in_use); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_mmu_as_released); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_total_alloc_pages_change); -+ -+void kbase_trace_mali_pm_status(u32 event, u64 value) -+{ -+ trace_mali_pm_status(event, value); -+} -+ -+void kbase_trace_mali_pm_power_off(u32 event, u64 value) -+{ -+ trace_mali_pm_power_off(event, value); -+} -+ -+void kbase_trace_mali_pm_power_on(u32 event, u64 value) -+{ -+ trace_mali_pm_power_on(event, value); -+} -+ -+void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id) -+{ -+ trace_mali_job_slots_event(event, (kctx != NULL ? kctx->tgid : 0), (kctx != NULL ? kctx->pid : 0), atom_id); -+} -+ -+void kbase_trace_mali_page_fault_insert_pages(int event, u32 value) -+{ -+ trace_mali_page_fault_insert_pages(event, value); -+} -+ -+void kbase_trace_mali_mmu_as_in_use(int event) -+{ -+ trace_mali_mmu_as_in_use(event); -+} -+ -+void kbase_trace_mali_mmu_as_released(int event) -+{ -+ trace_mali_mmu_as_released(event); -+} -+ -+void kbase_trace_mali_total_alloc_pages_change(long long int event) -+{ -+ trace_mali_total_alloc_pages_change(event); -+} -+#endif /* CONFIG_MALI_GATOR_SUPPORT */ -+#ifdef CONFIG_MALI_SYSTEM_TRACE -+#include "mali_linux_kbase_trace.h" -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c -new file mode 100755 -index 000000000..ce0048414 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c -@@ -0,0 +1,208 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+ -+#include "mali_kbase_ctx_sched.h" -+ -+int kbase_ctx_sched_init(struct kbase_device *kbdev) -+{ -+ int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; -+ -+ /* These two must be recalculated if nr_hw_address_spaces changes -+ * (e.g. for HW workarounds) */ -+ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { -+ bool use_workaround; -+ -+ use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; -+ if (use_workaround) { -+ dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); -+ kbdev->nr_user_address_spaces = 1; -+ } -+ } -+ -+ kbdev->as_free = as_present; /* All ASs initially free */ -+ -+ memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); -+ -+ return 0; -+} -+ -+void kbase_ctx_sched_term(struct kbase_device *kbdev) -+{ -+ s8 i; -+ -+ /* Sanity checks */ -+ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { -+ WARN_ON(kbdev->as_to_kctx[i] != NULL); -+ WARN_ON(!(kbdev->as_free & (1u << i))); -+ } -+} -+ -+/* kbasep_ctx_sched_find_as_for_ctx - Find a free address space -+ * -+ * @kbdev: The context for which to find a free address space -+ * -+ * Return: A valid AS if successful, otherwise KBASEP_AS_NR_INVALID -+ * -+ * This function returns an address space available for use. It would prefer -+ * returning an AS that has been previously assigned to the context to -+ * avoid having to reprogram the MMU. -+ */ -+static int kbasep_ctx_sched_find_as_for_ctx(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ int free_as; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* First check if the previously assigned AS is available */ -+ if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && -+ (kbdev->as_free & (1u << kctx->as_nr))) -+ return kctx->as_nr; -+ -+ /* The previously assigned AS was taken, we'll be returning any free -+ * AS at this point. -+ */ -+ free_as = ffs(kbdev->as_free) - 1; -+ if (free_as >= 0 && free_as < kbdev->nr_hw_address_spaces) -+ return free_as; -+ -+ return KBASEP_AS_NR_INVALID; -+} -+ -+int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ lockdep_assert_held(&kbdev->mmu_hw_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ON(!kbdev->pm.backend.gpu_powered); -+ -+ if (atomic_inc_return(&kctx->refcount) == 1) { -+ int const free_as = kbasep_ctx_sched_find_as_for_ctx(kctx); -+ -+ if (free_as != KBASEP_AS_NR_INVALID) { -+ kbdev->as_free &= ~(1u << free_as); -+ /* Only program the MMU if the context has not been -+ * assigned the same address space before. -+ */ -+ if (free_as != kctx->as_nr) { -+ struct kbase_context *const prev_kctx = -+ kbdev->as_to_kctx[free_as]; -+ -+ if (prev_kctx) { -+ WARN_ON(atomic_read(&prev_kctx->refcount) != 0); -+ kbase_mmu_disable(prev_kctx); -+ prev_kctx->as_nr = KBASEP_AS_NR_INVALID; -+ } -+ -+ kctx->as_nr = free_as; -+ kbdev->as_to_kctx[free_as] = kctx; -+ kbase_mmu_update(kctx); -+ } -+ } else { -+ atomic_dec(&kctx->refcount); -+ -+ /* Failed to find an available address space, we must -+ * be returning an error at this point. -+ */ -+ WARN_ON(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ } -+ } -+ -+ return kctx->as_nr; -+} -+ -+int kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ WARN_ON(atomic_read(&kctx->refcount) == 0); -+ if (atomic_read(&kctx->refcount) == 0) -+ return -1; -+ -+ WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID); -+ WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); -+ -+ atomic_inc(&kctx->refcount); -+ -+ return 0; -+} -+ -+void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (atomic_dec_return(&kctx->refcount) == 0) -+ kbdev->as_free |= (1u << kctx->as_nr); -+} -+ -+void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ lockdep_assert_held(&kbdev->mmu_hw_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ON(atomic_read(&kctx->refcount) != 0); -+ -+ if (kctx->as_nr != KBASEP_AS_NR_INVALID) { -+ if (kbdev->pm.backend.gpu_powered) -+ kbase_mmu_disable(kctx); -+ -+ kbdev->as_to_kctx[kctx->as_nr] = NULL; -+ kctx->as_nr = KBASEP_AS_NR_INVALID; -+ } -+} -+ -+void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) -+{ -+ s8 i; -+ -+ lockdep_assert_held(&kbdev->mmu_hw_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ON(!kbdev->pm.backend.gpu_powered); -+ -+ for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { -+ struct kbase_context *kctx; -+ -+ kctx = kbdev->as_to_kctx[i]; -+ if (kctx) { -+ if (atomic_read(&kctx->refcount)) { -+ WARN_ON(kctx->as_nr != i); -+ -+ kbase_mmu_update(kctx); -+ } else { -+ /* This context might have been assigned an -+ * AS before, clear it. -+ */ -+ kbdev->as_to_kctx[kctx->as_nr] = NULL; -+ kctx->as_nr = KBASEP_AS_NR_INVALID; -+ } -+ } else { -+ kbase_mmu_disable_as(kbdev, i); -+ } -+ } -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h -new file mode 100755 -index 000000000..47474fecc ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h -@@ -0,0 +1,134 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_CTX_SCHED_H_ -+#define _KBASE_CTX_SCHED_H_ -+ -+#include -+ -+/* The Context Scheduler manages address space assignment and reference -+ * counting to kbase_context. The interface has been designed to minimise -+ * interactions between the Job Scheduler and Power Management/MMU to support -+ * both the existing Job Scheduler and Command Stream Frontend interface. -+ * -+ * The initial implementation of the Context Scheduler does not schedule -+ * contexts. Instead it relies on the Job Scheduler/CSF to make decisions of -+ * when to schedule/evict contexts if address spaces are starved. In the -+ * future, once an interface between the CS and JS/CSF have been devised to -+ * provide enough information about how each context is consuming GPU resources, -+ * those decisions can be made in the CS itself, thereby reducing duplicated -+ * code. -+ */ -+ -+/* base_ctx_sched_init - Initialise the context scheduler -+ * -+ * @kbdev: The device for which the context scheduler needs to be -+ * initialised -+ * -+ * Return: 0 for success, otherwise failure -+ * -+ * This must be called during device initilisation. The number of hardware -+ * address spaces must already be established before calling this function. -+ */ -+int kbase_ctx_sched_init(struct kbase_device *kbdev); -+ -+/* base_ctx_sched_term - Terminate the context scheduler -+ * -+ * @kbdev: The device for which the context scheduler needs to be -+ * terminated -+ * -+ * This must be called during device termination after all contexts have been -+ * destroyed. -+ */ -+void kbase_ctx_sched_term(struct kbase_device *kbdev); -+ -+/* kbase_ctx_sched_retain_ctx - Retain a reference to the @ref kbase_context -+ * -+ * @kctx: The context to which to retain a reference -+ * -+ * Return: The address space that the context has been assigned to or -+ * KBASEP_AS_NR_INVALID if no address space was available. -+ * -+ * This function should be called whenever an address space should be assigned -+ * to a context and programmed onto the MMU. It should typically be called -+ * when jobs are ready to be submitted to the GPU. -+ * -+ * It can be called as many times as necessary. The address space will be -+ * assigned to the context for as long as there is a reference to said context. -+ * -+ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be -+ * held whilst calling this function. -+ */ -+int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx); -+ -+/* kbase_ctx_sched_retain_ctx_refcount -+ * -+ * @kctx: The context to which to retain a reference -+ * -+ * This function only retains a reference to the context. It must be called -+ * only when the context already has a reference. -+ * -+ * This is typically called inside an atomic session where we know the context -+ * is already scheduled in but want to take an extra reference to ensure that -+ * it doesn't get descheduled. -+ * -+ * The kbase_device::hwaccess_lock must be held whilst calling this function -+ * @return -+ * è‹¥æˆåŠŸ, 返回 0; -+ * è‹¥ *kctx 状æ€å¼‚常, 返回 -1. -+ */ -+int kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx); -+ -+/* kbase_ctx_sched_release_ctx - Release a reference to the @ref kbase_context -+ * -+ * @kctx: The context from which to release a reference -+ * -+ * This function should be called whenever an address space could be unassigned -+ * from a context. When there are no more references to said context, the -+ * address space previously assigned to this context shall be reassigned to -+ * other contexts as needed. -+ * -+ * The kbase_device::hwaccess_lock must be held whilst calling this function -+ */ -+void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); -+ -+/* kbase_ctx_sched_remove_ctx - Unassign previously assigned address space -+ * -+ * @kctx: The context to be removed -+ * -+ * This function should be called when a context is being destroyed. The -+ * context must no longer have any reference. If it has been assigned an -+ * address space before then the AS will be unprogrammed. -+ * -+ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be -+ * held whilst calling this function. -+ */ -+void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); -+ -+/* kbase_ctx_sched_restore_all_as - Reprogram all address spaces -+ * -+ * @kbdev: The device for which address spaces to be reprogrammed -+ * -+ * This function shall reprogram all address spaces previously assigned to -+ * contexts. It can be used after the GPU is reset. -+ * -+ * The kbase_device::mmu_hw_mutex and kbase_device::hwaccess_lock locks must be -+ * held whilst calling this function. -+ */ -+void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_CTX_SCHED_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug.c b/drivers/gpu/arm/midgard/mali_kbase_debug.c -new file mode 100755 -index 000000000..fb57ac2e3 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_debug.c -@@ -0,0 +1,39 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+ -+static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { -+ NULL, -+ NULL -+}; -+ -+void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param) -+{ -+ kbasep_debug_assert_registered_cb.func = func; -+ kbasep_debug_assert_registered_cb.param = param; -+} -+ -+void kbasep_debug_assert_call_hook(void) -+{ -+ if (kbasep_debug_assert_registered_cb.func != NULL) -+ kbasep_debug_assert_registered_cb.func(kbasep_debug_assert_registered_cb.param); -+} -+KBASE_EXPORT_SYMBOL(kbasep_debug_assert_call_hook); -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug.h b/drivers/gpu/arm/midgard/mali_kbase_debug.h -new file mode 100755 -index 000000000..5fff2892b ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_debug.h -@@ -0,0 +1,164 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_DEBUG_H -+#define _KBASE_DEBUG_H -+ -+#include -+ -+/** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ -+#define KBASE_DEBUG_SKIP_TRACE 0 -+ -+/** @brief If different from 0, the trace will only contain the file and line. */ -+#define KBASE_DEBUG_SKIP_FUNCTION_NAME 0 -+ -+/** @brief Disable the asserts tests if set to 1. Default is to disable the asserts in release. */ -+#ifndef KBASE_DEBUG_DISABLE_ASSERTS -+#ifdef CONFIG_MALI_DEBUG -+#define KBASE_DEBUG_DISABLE_ASSERTS 0 -+#else -+#define KBASE_DEBUG_DISABLE_ASSERTS 1 -+#endif -+#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ -+ -+/** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ -+typedef void (kbase_debug_assert_hook) (void *); -+ -+struct kbasep_debug_assert_cb { -+ kbase_debug_assert_hook *func; -+ void *param; -+}; -+ -+/** -+ * @def KBASEP_DEBUG_PRINT_TRACE -+ * @brief Private macro containing the format of the trace to display before every message -+ * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME -+ */ -+#if !KBASE_DEBUG_SKIP_TRACE -+#define KBASEP_DEBUG_PRINT_TRACE \ -+ "In file: " __FILE__ " line: " CSTD_STR2(__LINE__) -+#if !KBASE_DEBUG_SKIP_FUNCTION_NAME -+#define KBASEP_DEBUG_PRINT_FUNCTION __func__ -+#else -+#define KBASEP_DEBUG_PRINT_FUNCTION "" -+#endif -+#else -+#define KBASEP_DEBUG_PRINT_TRACE "" -+#endif -+ -+/** -+ * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) -+ * @brief (Private) system printing function associated to the @see KBASE_DEBUG_ASSERT_MSG event. -+ * @param trace location in the code from where the message is printed -+ * @param function function from where the message is printed -+ * @param ... Format string followed by format arguments. -+ * @note function parameter cannot be concatenated with other strings -+ */ -+/* Select the correct system output function*/ -+#ifdef CONFIG_MALI_DEBUG -+#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ -+ do { \ -+ pr_err("Mali: %s function:%s ", trace, function);\ -+ pr_err(__VA_ARGS__);\ -+ pr_err("\n");\ -+ } while (false) -+#else -+#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() -+#endif -+ -+#ifdef CONFIG_MALI_DEBUG -+#define KBASE_CALL_ASSERT_HOOK() kbasep_debug_assert_call_hook() -+#else -+#define KBASE_CALL_ASSERT_HOOK() CSTD_NOP() -+#endif -+ -+/** -+ * @def KBASE_DEBUG_ASSERT(expr) -+ * @brief Calls @see KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false -+ * -+ * @note This macro does nothing if the flag @see KBASE_DEBUG_DISABLE_ASSERTS is set to 1 -+ * -+ * @param expr Boolean expression -+ */ -+#define KBASE_DEBUG_ASSERT(expr) \ -+ KBASE_DEBUG_ASSERT_MSG(expr, #expr) -+ -+#if KBASE_DEBUG_DISABLE_ASSERTS -+#define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() -+#else -+ /** -+ * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) -+ * @brief Calls @see KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false -+ * -+ * @note This macro does nothing if the flag @see KBASE_DEBUG_DISABLE_ASSERTS is set to 1 -+ * -+ * @param expr Boolean expression -+ * @param ... Message to display when @a expr is false, as a format string followed by format arguments. -+ */ -+#define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ -+ do { \ -+ if (!(expr)) { \ -+ KBASEP_DEBUG_ASSERT_OUT(KBASEP_DEBUG_PRINT_TRACE, KBASEP_DEBUG_PRINT_FUNCTION, __VA_ARGS__);\ -+ KBASE_CALL_ASSERT_HOOK();\ -+ BUG();\ -+ } \ -+ } while (false) -+#endif /* KBASE_DEBUG_DISABLE_ASSERTS */ -+ -+/** -+ * @def KBASE_DEBUG_CODE( X ) -+ * @brief Executes the code inside the macro only in debug mode -+ * -+ * @param X Code to compile only in debug mode. -+ */ -+#ifdef CONFIG_MALI_DEBUG -+#define KBASE_DEBUG_CODE(X) X -+#else -+#define KBASE_DEBUG_CODE(X) CSTD_NOP() -+#endif /* CONFIG_MALI_DEBUG */ -+ -+/** @} */ -+ -+/** -+ * @brief Register a function to call on ASSERT -+ * -+ * Such functions will \b only be called during Debug mode, and for debugging -+ * features \b only. Do not rely on them to be called in general use. -+ * -+ * To disable the hook, supply NULL to \a func. -+ * -+ * @note This function is not thread-safe, and should only be used to -+ * register/deregister once in the module's lifetime. -+ * -+ * @param[in] func the function to call when an assert is triggered. -+ * @param[in] param the parameter to pass to \a func when calling it -+ */ -+void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); -+ -+/** -+ * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() -+ * -+ * @note This function is not thread-safe with respect to multiple threads -+ * registering functions and parameters with -+ * kbase_debug_assert_register_hook(). Otherwise, thread safety is the -+ * responsibility of the registered hook. -+ */ -+void kbasep_debug_assert_call_hook(void); -+ -+#endif /* _KBASE_DEBUG_H */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c -new file mode 100755 -index 000000000..f29430ddf ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c -@@ -0,0 +1,499 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+ -+#ifdef CONFIG_DEBUG_FS -+ -+static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) -+{ -+ struct list_head *event_list = &kbdev->job_fault_event_list; -+ unsigned long flags; -+ bool ret; -+ -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ ret = !list_empty(event_list); -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ -+ return ret; -+} -+ -+static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct list_head *event_list = &kctx->kbdev->job_fault_event_list; -+ struct base_job_fault_event *event; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ if (list_empty(event_list)) { -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ return true; -+ } -+ list_for_each_entry(event, event_list, head) { -+ if (event->katom->kctx == kctx) { -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, -+ flags); -+ return false; -+ } -+ } -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ return true; -+} -+ -+/* wait until the fault happen and copy the event */ -+static int kbase_job_fault_event_wait(struct kbase_device *kbdev, -+ struct base_job_fault_event *event) -+{ -+ struct list_head *event_list = &kbdev->job_fault_event_list; -+ struct base_job_fault_event *event_in; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ if (list_empty(event_list)) { -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ if (wait_event_interruptible(kbdev->job_fault_wq, -+ kbase_is_job_fault_event_pending(kbdev))) -+ return -ERESTARTSYS; -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ } -+ -+ event_in = list_entry(event_list->next, -+ struct base_job_fault_event, head); -+ event->event_code = event_in->event_code; -+ event->katom = event_in->katom; -+ -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ -+ return 0; -+ -+} -+ -+/* remove the event from the queue */ -+static struct base_job_fault_event *kbase_job_fault_event_dequeue( -+ struct kbase_device *kbdev, struct list_head *event_list) -+{ -+ struct base_job_fault_event *event; -+ -+ event = list_entry(event_list->next, -+ struct base_job_fault_event, head); -+ list_del(event_list->next); -+ -+ return event; -+ -+} -+ -+/* Remove all the following atoms after the failed atom in the same context -+ * Call the postponed bottom half of job done. -+ * Then, this context could be rescheduled. -+ */ -+static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) -+{ -+ struct list_head *event_list = &kctx->job_fault_resume_event_list; -+ -+ while (!list_empty(event_list)) { -+ struct base_job_fault_event *event; -+ -+ event = kbase_job_fault_event_dequeue(kctx->kbdev, -+ &kctx->job_fault_resume_event_list); -+ kbase_jd_done_worker(&event->katom->work); -+ } -+ -+} -+ -+/* Remove all the failed atoms that belong to different contexts -+ * Resume all the contexts that were suspend due to failed job -+ */ -+static void kbase_job_fault_event_cleanup(struct kbase_device *kbdev) -+{ -+ struct list_head *event_list = &kbdev->job_fault_event_list; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ while (!list_empty(event_list)) { -+ kbase_job_fault_event_dequeue(kbdev, event_list); -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ wake_up(&kbdev->job_fault_resume_wq); -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ } -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+} -+ -+static void kbase_job_fault_resume_worker(struct work_struct *data) -+{ -+ struct base_job_fault_event *event = container_of(data, -+ struct base_job_fault_event, job_fault_work); -+ struct kbase_context *kctx; -+ struct kbase_jd_atom *katom; -+ -+ katom = event->katom; -+ kctx = katom->kctx; -+ -+ dev_info(kctx->kbdev->dev, "Job dumping wait\n"); -+ -+ /* When it was waked up, it need to check if queue is empty or the -+ * failed atom belongs to different context. If yes, wake up. Both -+ * of them mean the failed job has been dumped. Please note, it -+ * should never happen that the job_fault_event_list has the two -+ * atoms belong to the same context. -+ */ -+ wait_event(kctx->kbdev->job_fault_resume_wq, -+ kbase_ctx_has_no_event_pending(kctx)); -+ -+ atomic_set(&kctx->job_fault_count, 0); -+ kbase_jd_done_worker(&katom->work); -+ -+ /* In case the following atoms were scheduled during failed job dump -+ * the job_done_worker was held. We need to rerun it after the dump -+ * was finished -+ */ -+ kbase_job_fault_resume_event_cleanup(kctx); -+ -+ dev_info(kctx->kbdev->dev, "Job dumping finish, resume scheduler\n"); -+} -+ -+static struct base_job_fault_event *kbase_job_fault_event_queue( -+ struct list_head *event_list, -+ struct kbase_jd_atom *atom, -+ u32 completion_code) -+{ -+ struct base_job_fault_event *event; -+ -+ event = &atom->fault_event; -+ -+ event->katom = atom; -+ event->event_code = completion_code; -+ -+ list_add_tail(&event->head, event_list); -+ -+ return event; -+ -+} -+ -+static void kbase_job_fault_event_post(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, u32 completion_code) -+{ -+ struct base_job_fault_event *event; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, -+ katom, completion_code); -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ -+ wake_up_interruptible(&kbdev->job_fault_wq); -+ -+ INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); -+ queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); -+ -+ dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", -+ katom->kctx->tgid, katom->kctx->id); -+ -+} -+ -+/* -+ * This function will process the job fault -+ * Get the register copy -+ * Send the failed job dump event -+ * Create a Wait queue to wait until the job dump finish -+ */ -+ -+bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, -+ u32 completion_code) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ /* Check if dumping is in the process -+ * only one atom of each context can be dumped at the same time -+ * If the atom belongs to different context, it can be dumped -+ */ -+ if (atomic_read(&kctx->job_fault_count) > 0) { -+ kbase_job_fault_event_queue( -+ &kctx->job_fault_resume_event_list, -+ katom, completion_code); -+ dev_info(kctx->kbdev->dev, "queue:%d\n", -+ kbase_jd_atom_id(kctx, katom)); -+ return true; -+ } -+ -+ if (kctx->kbdev->job_fault_debug == true) { -+ -+ if (completion_code != BASE_JD_EVENT_DONE) { -+ -+ if (kbase_job_fault_get_reg_snapshot(kctx) == false) { -+ dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); -+ return false; -+ } -+ -+ kbase_job_fault_event_post(kctx->kbdev, katom, -+ completion_code); -+ atomic_inc(&kctx->job_fault_count); -+ dev_info(kctx->kbdev->dev, "post:%d\n", -+ kbase_jd_atom_id(kctx, katom)); -+ return true; -+ -+ } -+ } -+ return false; -+ -+} -+ -+static int debug_job_fault_show(struct seq_file *m, void *v) -+{ -+ struct kbase_device *kbdev = m->private; -+ struct base_job_fault_event *event = (struct base_job_fault_event *)v; -+ struct kbase_context *kctx = event->katom->kctx; -+ int i; -+ -+ dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", -+ kctx->tgid, kctx->id, event->reg_offset); -+ -+ if (kctx->reg_dump == NULL) { -+ dev_warn(kbdev->dev, "reg dump is NULL"); -+ return -1; -+ } -+ -+ if (kctx->reg_dump[event->reg_offset] == -+ REGISTER_DUMP_TERMINATION_FLAG) { -+ /* Return the error here to stop the read. And the -+ * following next() will not be called. The stop can -+ * get the real event resource and release it -+ */ -+ return -1; -+ } -+ -+ if (event->reg_offset == 0) -+ seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); -+ -+ for (i = 0; i < 50; i++) { -+ if (kctx->reg_dump[event->reg_offset] == -+ REGISTER_DUMP_TERMINATION_FLAG) { -+ break; -+ } -+ seq_printf(m, "%08x: %08x\n", -+ kctx->reg_dump[event->reg_offset], -+ kctx->reg_dump[1+event->reg_offset]); -+ event->reg_offset += 2; -+ -+ } -+ -+ -+ return 0; -+} -+static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) -+{ -+ struct kbase_device *kbdev = m->private; -+ struct base_job_fault_event *event = (struct base_job_fault_event *)v; -+ -+ dev_info(kbdev->dev, "debug job fault seq next:%d, %d", -+ event->reg_offset, (int)*pos); -+ -+ return event; -+} -+ -+static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) -+{ -+ struct kbase_device *kbdev = m->private; -+ struct base_job_fault_event *event; -+ -+ dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); -+ -+ /* The condition is trick here. It needs make sure the -+ * fault hasn't happened and the dumping hasn't been started, -+ * or the dumping has finished -+ */ -+ if (*pos == 0) { -+ event = kmalloc(sizeof(*event), GFP_KERNEL); -+ if (!event) -+ return NULL; -+ event->reg_offset = 0; -+ if (kbase_job_fault_event_wait(kbdev, event)) { -+ kfree(event); -+ return NULL; -+ } -+ -+ /* The cache flush workaround is called in bottom half of -+ * job done but we delayed it. Now we should clean cache -+ * earlier. Then the GPU memory dump should be correct. -+ */ -+ kbase_backend_cacheclean(kbdev, event->katom); -+ } else -+ return NULL; -+ -+ return event; -+} -+ -+static void debug_job_fault_stop(struct seq_file *m, void *v) -+{ -+ struct kbase_device *kbdev = m->private; -+ -+ /* here we wake up the kbase_jd_done_worker after stop, it needs -+ * get the memory dump before the register dump in debug daemon, -+ * otherwise, the memory dump may be incorrect. -+ */ -+ -+ if (v != NULL) { -+ kfree(v); -+ dev_info(kbdev->dev, "debug job fault seq stop stage 1"); -+ -+ } else { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); -+ if (!list_empty(&kbdev->job_fault_event_list)) { -+ kbase_job_fault_event_dequeue(kbdev, -+ &kbdev->job_fault_event_list); -+ wake_up(&kbdev->job_fault_resume_wq); -+ } -+ spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); -+ dev_info(kbdev->dev, "debug job fault seq stop stage 2"); -+ } -+ -+} -+ -+static const struct seq_operations ops = { -+ .start = debug_job_fault_start, -+ .next = debug_job_fault_next, -+ .stop = debug_job_fault_stop, -+ .show = debug_job_fault_show, -+}; -+ -+static int debug_job_fault_open(struct inode *in, struct file *file) -+{ -+ struct kbase_device *kbdev = in->i_private; -+ -+ seq_open(file, &ops); -+ -+ ((struct seq_file *)file->private_data)->private = kbdev; -+ dev_info(kbdev->dev, "debug job fault seq open"); -+ -+ kbdev->job_fault_debug = true; -+ -+ return 0; -+ -+} -+ -+static int debug_job_fault_release(struct inode *in, struct file *file) -+{ -+ struct kbase_device *kbdev = in->i_private; -+ -+ seq_release(in, file); -+ -+ kbdev->job_fault_debug = false; -+ -+ /* Clean the unprocessed job fault. After that, all the suspended -+ * contexts could be rescheduled. -+ */ -+ kbase_job_fault_event_cleanup(kbdev); -+ -+ dev_info(kbdev->dev, "debug job fault seq close"); -+ -+ return 0; -+} -+ -+static const struct file_operations kbasep_debug_job_fault_fops = { -+ .open = debug_job_fault_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = debug_job_fault_release, -+}; -+ -+/* -+ * Initialize debugfs entry for job fault dump -+ */ -+void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("job_fault", S_IRUGO, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_debug_job_fault_fops); -+} -+ -+ -+int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) -+{ -+ -+ INIT_LIST_HEAD(&kbdev->job_fault_event_list); -+ -+ init_waitqueue_head(&(kbdev->job_fault_wq)); -+ init_waitqueue_head(&(kbdev->job_fault_resume_wq)); -+ spin_lock_init(&kbdev->job_fault_event_lock); -+ -+ kbdev->job_fault_resume_workq = alloc_workqueue( -+ "kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); -+ if (!kbdev->job_fault_resume_workq) -+ return -ENOMEM; -+ -+ kbdev->job_fault_debug = false; -+ -+ return 0; -+} -+ -+/* -+ * Release the relevant resource per device -+ */ -+void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) -+{ -+ destroy_workqueue(kbdev->job_fault_resume_workq); -+} -+ -+ -+/* -+ * Initialize the relevant data structure per context -+ */ -+void kbase_debug_job_fault_context_init(struct kbase_context *kctx) -+{ -+ -+ /* We need allocate double size register range -+ * Because this memory will keep the register address and value -+ */ -+ kctx->reg_dump = vmalloc(0x4000 * 2); -+ if (kctx->reg_dump == NULL) -+ return; -+ -+ if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { -+ vfree(kctx->reg_dump); -+ kctx->reg_dump = NULL; -+ } -+ INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); -+ atomic_set(&kctx->job_fault_count, 0); -+ -+} -+ -+/* -+ * release the relevant resource per context -+ */ -+void kbase_debug_job_fault_context_term(struct kbase_context *kctx) -+{ -+ vfree(kctx->reg_dump); -+} -+ -+#else /* CONFIG_DEBUG_FS */ -+ -+int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) -+{ -+ kbdev->job_fault_debug = false; -+ -+ return 0; -+} -+ -+void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) -+{ -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h -new file mode 100755 -index 000000000..a2bf8983c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h -@@ -0,0 +1,96 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_DEBUG_JOB_FAULT_H -+#define _KBASE_DEBUG_JOB_FAULT_H -+ -+#include -+#include -+ -+#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF -+ -+/** -+ * kbase_debug_job_fault_dev_init - Create the fault event wait queue -+ * per device and initialize the required lists. -+ * @kbdev: Device pointer -+ * -+ * Return: Zero on success or a negative error code. -+ */ -+int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs -+ * @kbdev: Device pointer -+ */ -+void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_debug_job_fault_dev_term - Clean up resources created in -+ * kbase_debug_job_fault_dev_init. -+ * @kbdev: Device pointer -+ */ -+void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_debug_job_fault_context_init - Initialize the relevant -+ * data structure per context -+ * @kctx: KBase context pointer -+ */ -+void kbase_debug_job_fault_context_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_debug_job_fault_context_term - Release the relevant -+ * resource per context -+ * @kctx: KBase context pointer -+ */ -+void kbase_debug_job_fault_context_term(struct kbase_context *kctx); -+ -+/** -+ * kbase_debug_job_fault_process - Process the failed job. -+ * It will send a event and wake up the job fault waiting queue -+ * Then create a work queue to wait for job dump finish -+ * This function should be called in the interrupt handler and before -+ * jd_done that make sure the jd_done_worker will be delayed until the -+ * job dump finish -+ * @katom: The failed atom pointer -+ * @completion_code: the job status -+ * @return true if dump is going on -+ */ -+bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, -+ u32 completion_code); -+ -+ -+/** -+ * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers -+ * address during the job fault process, the relevant registers will -+ * be saved when a job fault happen -+ * @kctx: KBase context pointer -+ * @reg_range: Maximum register address space -+ * @return true if initializing successfully -+ */ -+bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, -+ int reg_range); -+ -+/** -+ * kbase_job_fault_get_reg_snapshot - Read the interested registers for -+ * failed job dump -+ * @kctx: KBase context pointer -+ * @return true if getting registers successfully -+ */ -+bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); -+ -+#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c -new file mode 100755 -index 000000000..6f2cbdf57 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c -@@ -0,0 +1,306 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Debugfs interface to dump the memory visible to the GPU -+ */ -+ -+#include "mali_kbase_debug_mem_view.h" -+#include "mali_kbase.h" -+ -+#include -+#include -+ -+#ifdef CONFIG_DEBUG_FS -+ -+struct debug_mem_mapping { -+ struct list_head node; -+ -+ struct kbase_mem_phy_alloc *alloc; -+ unsigned long flags; -+ -+ u64 start_pfn; -+ size_t nr_pages; -+}; -+ -+struct debug_mem_data { -+ struct list_head mapping_list; -+ struct kbase_context *kctx; -+}; -+ -+struct debug_mem_seq_off { -+ struct list_head *lh; -+ size_t offset; -+}; -+ -+static void *debug_mem_start(struct seq_file *m, loff_t *_pos) -+{ -+ struct debug_mem_data *mem_data = m->private; -+ struct debug_mem_seq_off *data; -+ struct debug_mem_mapping *map; -+ loff_t pos = *_pos; -+ -+ list_for_each_entry(map, &mem_data->mapping_list, node) { -+ if (pos >= map->nr_pages) { -+ pos -= map->nr_pages; -+ } else { -+ data = kmalloc(sizeof(*data), GFP_KERNEL); -+ if (!data) -+ return NULL; -+ data->lh = &map->node; -+ data->offset = pos; -+ return data; -+ } -+ } -+ -+ /* Beyond the end */ -+ return NULL; -+} -+ -+static void debug_mem_stop(struct seq_file *m, void *v) -+{ -+ kfree(v); -+} -+ -+static void *debug_mem_next(struct seq_file *m, void *v, loff_t *pos) -+{ -+ struct debug_mem_data *mem_data = m->private; -+ struct debug_mem_seq_off *data = v; -+ struct debug_mem_mapping *map; -+ -+ map = list_entry(data->lh, struct debug_mem_mapping, node); -+ -+ if (data->offset < map->nr_pages - 1) { -+ data->offset++; -+ ++*pos; -+ return data; -+ } -+ -+ if (list_is_last(data->lh, &mem_data->mapping_list)) { -+ kfree(data); -+ return NULL; -+ } -+ -+ data->lh = data->lh->next; -+ data->offset = 0; -+ ++*pos; -+ -+ return data; -+} -+ -+static int debug_mem_show(struct seq_file *m, void *v) -+{ -+ struct debug_mem_data *mem_data = m->private; -+ struct debug_mem_seq_off *data = v; -+ struct debug_mem_mapping *map; -+ int i, j; -+ struct page *page; -+ uint32_t *mapping; -+ pgprot_t prot = PAGE_KERNEL; -+ -+ map = list_entry(data->lh, struct debug_mem_mapping, node); -+ -+ kbase_gpu_vm_lock(mem_data->kctx); -+ -+ if (data->offset >= map->alloc->nents) { -+ seq_printf(m, "%016llx: Unbacked page\n\n", (map->start_pfn + -+ data->offset) << PAGE_SHIFT); -+ goto out; -+ } -+ -+ if (!(map->flags & KBASE_REG_CPU_CACHED)) -+ prot = pgprot_writecombine(prot); -+ -+ page = pfn_to_page(PFN_DOWN(map->alloc->pages[data->offset])); -+ mapping = vmap(&page, 1, VM_MAP, prot); -+ if (!mapping) -+ goto out; -+ -+ for (i = 0; i < PAGE_SIZE; i += 4*sizeof(*mapping)) { -+ seq_printf(m, "%016llx:", i + ((map->start_pfn + -+ data->offset) << PAGE_SHIFT)); -+ -+ for (j = 0; j < 4*sizeof(*mapping); j += sizeof(*mapping)) -+ seq_printf(m, " %08x", mapping[(i+j)/sizeof(*mapping)]); -+ seq_putc(m, '\n'); -+ } -+ -+ vunmap(mapping); -+ -+ seq_putc(m, '\n'); -+ -+out: -+ kbase_gpu_vm_unlock(mem_data->kctx); -+ return 0; -+} -+ -+static const struct seq_operations ops = { -+ .start = debug_mem_start, -+ .next = debug_mem_next, -+ .stop = debug_mem_stop, -+ .show = debug_mem_show, -+}; -+ -+static int debug_mem_zone_open(struct rb_root *rbtree, -+ struct debug_mem_data *mem_data) -+{ -+ int ret = 0; -+ struct rb_node *p; -+ struct kbase_va_region *reg; -+ struct debug_mem_mapping *mapping; -+ -+ for (p = rb_first(rbtree); p; p = rb_next(p)) { -+ reg = rb_entry(p, struct kbase_va_region, rblink); -+ -+ if (reg->gpu_alloc == NULL) -+ /* Empty region - ignore */ -+ continue; -+ -+ mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); -+ if (!mapping) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ mapping->alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ mapping->start_pfn = reg->start_pfn; -+ mapping->nr_pages = reg->nr_pages; -+ mapping->flags = reg->flags; -+ list_add_tail(&mapping->node, &mem_data->mapping_list); -+ } -+ -+out: -+ return ret; -+} -+ -+static int debug_mem_open(struct inode *i, struct file *file) -+{ -+ struct file *kctx_file = i->i_private; -+ struct kbase_context *kctx = kctx_file->private_data; -+ struct debug_mem_data *mem_data; -+ int ret; -+ -+ ret = seq_open(file, &ops); -+ if (ret) -+ return ret; -+ -+ mem_data = kmalloc(sizeof(*mem_data), GFP_KERNEL); -+ if (!mem_data) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ mem_data->kctx = kctx; -+ -+ INIT_LIST_HEAD(&mem_data->mapping_list); -+ -+ get_file(kctx_file); -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); -+ if (0 != ret) { -+ kbase_gpu_vm_unlock(kctx); -+ goto out; -+ } -+ -+ ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); -+ if (0 != ret) { -+ kbase_gpu_vm_unlock(kctx); -+ goto out; -+ } -+ -+ ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); -+ if (0 != ret) { -+ kbase_gpu_vm_unlock(kctx); -+ goto out; -+ } -+ -+ kbase_gpu_vm_unlock(kctx); -+ -+ ((struct seq_file *)file->private_data)->private = mem_data; -+ -+ return 0; -+ -+out: -+ if (mem_data) { -+ while (!list_empty(&mem_data->mapping_list)) { -+ struct debug_mem_mapping *mapping; -+ -+ mapping = list_first_entry(&mem_data->mapping_list, -+ struct debug_mem_mapping, node); -+ kbase_mem_phy_alloc_put(mapping->alloc); -+ list_del(&mapping->node); -+ kfree(mapping); -+ } -+ fput(kctx_file); -+ kfree(mem_data); -+ } -+ seq_release(i, file); -+ return ret; -+} -+ -+static int debug_mem_release(struct inode *inode, struct file *file) -+{ -+ struct file *kctx_file = inode->i_private; -+ struct seq_file *sfile = file->private_data; -+ struct debug_mem_data *mem_data = sfile->private; -+ struct debug_mem_mapping *mapping; -+ -+ seq_release(inode, file); -+ -+ while (!list_empty(&mem_data->mapping_list)) { -+ mapping = list_first_entry(&mem_data->mapping_list, -+ struct debug_mem_mapping, node); -+ kbase_mem_phy_alloc_put(mapping->alloc); -+ list_del(&mapping->node); -+ kfree(mapping); -+ } -+ -+ kfree(mem_data); -+ -+ fput(kctx_file); -+ -+ return 0; -+} -+ -+static const struct file_operations kbase_debug_mem_view_fops = { -+ .open = debug_mem_open, -+ .release = debug_mem_release, -+ .read = seq_read, -+ .llseek = seq_lseek -+}; -+ -+/** -+ * kbase_debug_mem_view_init - Initialise the mem_view sysfs file -+ * @kctx_file: The /dev/mali0 file instance for the context -+ * -+ * This function creates a "mem_view" file which can be used to get a view of -+ * the context's memory as the GPU sees it (i.e. using the GPU's page tables). -+ * -+ * The file is cleaned up by a call to debugfs_remove_recursive() deleting the -+ * parent directory. -+ */ -+void kbase_debug_mem_view_init(struct file *kctx_file) -+{ -+ struct kbase_context *kctx = kctx_file->private_data; -+ -+ debugfs_create_file("mem_view", S_IRUGO, kctx->kctx_dentry, kctx_file, -+ &kbase_debug_mem_view_fops); -+} -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h -new file mode 100755 -index 000000000..20ab51a77 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h -@@ -0,0 +1,25 @@ -+/* -+ * -+ * (C) COPYRIGHT 2013-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_DEBUG_MEM_VIEW_H -+#define _KBASE_DEBUG_MEM_VIEW_H -+ -+#include -+ -+void kbase_debug_mem_view_init(struct file *kctx_file); -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_defs.h b/drivers/gpu/arm/midgard/mali_kbase_defs.h -new file mode 100755 -index 000000000..f8a6f33df ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_defs.h -@@ -0,0 +1,1602 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_defs.h -+ * -+ * Defintions (types, defines, etcs) common to Kbase. They are placed here to -+ * allow the hierarchy of header files to work. -+ */ -+ -+#ifndef _KBASE_DEFS_H_ -+#define _KBASE_DEFS_H_ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+ -+#ifdef CONFIG_MALI_FPGA_BUS_LOGGER -+#include -+#endif -+ -+ -+#ifdef CONFIG_KDS -+#include -+#endif /* CONFIG_KDS */ -+ -+#if defined(CONFIG_SYNC) -+#include -+#else -+#include "mali_kbase_fence_defs.h" -+#endif -+ -+#ifdef CONFIG_DEBUG_FS -+#include -+#endif /* CONFIG_DEBUG_FS */ -+ -+#ifdef CONFIG_MALI_DEVFREQ -+#include -+#endif /* CONFIG_MALI_DEVFREQ */ -+ -+#include -+#include -+#include -+ -+#if defined(CONFIG_PM) -+#define KBASE_PM_RUNTIME 1 -+#endif -+ -+/** Enable SW tracing when set */ -+#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE -+#define KBASE_TRACE_ENABLE 1 -+#endif -+ -+#ifndef KBASE_TRACE_ENABLE -+#ifdef CONFIG_MALI_DEBUG -+#define KBASE_TRACE_ENABLE 1 -+#else -+#define KBASE_TRACE_ENABLE 0 -+#endif /* CONFIG_MALI_DEBUG */ -+#endif /* KBASE_TRACE_ENABLE */ -+ -+/** Dump Job slot trace on error (only active if KBASE_TRACE_ENABLE != 0) */ -+#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1 -+ -+/** -+ * Number of milliseconds before resetting the GPU when a job cannot be "zapped" from the hardware. -+ * Note that the time is actually ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and the GPU -+ * actually being reset to give other contexts time for their jobs to be soft-stopped and removed from the hardware -+ * before resetting. -+ */ -+#define ZAP_TIMEOUT 1000 -+ -+/** Number of milliseconds before we time out on a GPU soft/hard reset */ -+#define RESET_TIMEOUT 500 -+ -+/** -+ * Prevent soft-stops from occuring in scheduling situations -+ * -+ * This is not due to HW issues, but when scheduling is desired to be more predictable. -+ * -+ * Therefore, soft stop may still be disabled due to HW issues. -+ * -+ * @note Soft stop will still be used for non-scheduling purposes e.g. when terminating a context. -+ * -+ * @note if not in use, define this value to 0 instead of \#undef'ing it -+ */ -+#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 -+ -+/** -+ * Prevent hard-stops from occuring in scheduling situations -+ * -+ * This is not due to HW issues, but when scheduling is desired to be more predictable. -+ * -+ * @note Hard stop will still be used for non-scheduling purposes e.g. when terminating a context. -+ * -+ * @note if not in use, define this value to 0 instead of \#undef'ing it -+ */ -+#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 -+ -+/** -+ * The maximum number of Job Slots to support in the Hardware. -+ * -+ * You can optimize this down if your target devices will only ever support a -+ * small number of job slots. -+ */ -+#define BASE_JM_MAX_NR_SLOTS 3 -+ -+/** -+ * The maximum number of Address Spaces to support in the Hardware. -+ * -+ * You can optimize this down if your target devices will only ever support a -+ * small number of Address Spaces -+ */ -+#define BASE_MAX_NR_AS 16 -+ -+/* mmu */ -+#define MIDGARD_MMU_VA_BITS 48 -+ -+#if MIDGARD_MMU_VA_BITS > 39 -+#define MIDGARD_MMU_TOPLEVEL 0 -+#else -+#define MIDGARD_MMU_TOPLEVEL 1 -+#endif -+ -+#define MIDGARD_MMU_BOTTOMLEVEL 3 -+ -+#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR) -+ -+/** setting in kbase_context::as_nr that indicates it's invalid */ -+#define KBASEP_AS_NR_INVALID (-1) -+ -+#define KBASE_LOCK_REGION_MAX_SIZE (63) -+#define KBASE_LOCK_REGION_MIN_SIZE (11) -+ -+#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ -+#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) -+#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1) -+ -+#include "mali_kbase_js_defs.h" -+#include "mali_kbase_hwaccess_defs.h" -+ -+#define KBASEP_FORCE_REPLAY_DISABLED 0 -+ -+/* Maximum force replay limit when randomization is enabled */ -+#define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16 -+ -+/** Atom has been previously soft-stoppped */ -+#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1) -+/** Atom has been previously retried to execute */ -+#define KBASE_KATOM_FLAGS_RERUN (1<<2) -+#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3) -+/** Atom has been previously hard-stopped. */ -+#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4) -+/** Atom has caused us to enter disjoint state */ -+#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5) -+/* Atom blocked on cross-slot dependency */ -+#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7) -+/* Atom has fail dependency on cross-slot dependency */ -+#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8) -+/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ -+#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9) -+/* Atom is currently holding a context reference */ -+#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10) -+/* Atom requires GPU to be in protected mode */ -+#define KBASE_KATOM_FLAG_PROTECTED (1<<11) -+/* Atom has been stored in runnable_tree */ -+#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12) -+ -+/* SW related flags about types of JS_COMMAND action -+ * NOTE: These must be masked off by JS_COMMAND_MASK */ -+ -+/** This command causes a disjoint event */ -+#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 -+ -+/** Bitmask of all SW related flags */ -+#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) -+ -+#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) -+#error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks -+#endif -+ -+/** Soft-stop command that causes a Disjoint event. This of course isn't -+ * entirely masked off by JS_COMMAND_MASK */ -+#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \ -+ (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) -+ -+#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT -+ -+/* Serialize atoms within a slot (ie only one atom per job slot) */ -+#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) -+/* Serialize atoms between slots (ie only one job slot running at any time) */ -+#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) -+/* Reset the GPU after each atom completion */ -+#define KBASE_SERIALIZE_RESET (1 << 2) -+ -+#ifdef CONFIG_DEBUG_FS -+struct base_job_fault_event { -+ -+ u32 event_code; -+ struct kbase_jd_atom *katom; -+ struct work_struct job_fault_work; -+ struct list_head head; -+ int reg_offset; -+}; -+ -+#endif -+ -+struct kbase_jd_atom_dependency { -+ struct kbase_jd_atom *atom; -+ u8 dep_type; -+}; -+ -+/** -+ * struct kbase_io_access - holds information about 1 register access -+ * -+ * @addr: first bit indicates r/w (r=0, w=1) -+ * @value: value written or read -+ */ -+struct kbase_io_access { -+ uintptr_t addr; -+ u32 value; -+}; -+ -+/** -+ * struct kbase_io_history - keeps track of all recent register accesses -+ * -+ * @enabled: true if register accesses are recorded, false otherwise -+ * @lock: spinlock protecting kbase_io_access array -+ * @count: number of registers read/written -+ * @size: number of elements in kbase_io_access array -+ * @buf: array of kbase_io_access -+ */ -+struct kbase_io_history { -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) -+ bool enabled; -+#else -+ u32 enabled; -+#endif -+ -+ spinlock_t lock; -+ size_t count; -+ u16 size; -+ struct kbase_io_access *buf; -+}; -+ -+/** -+ * @brief The function retrieves a read-only reference to the atom field from -+ * the kbase_jd_atom_dependency structure -+ * -+ * @param[in] dep kbase jd atom dependency. -+ * -+ * @return readonly reference to dependent ATOM. -+ */ -+static inline const struct kbase_jd_atom * kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) -+{ -+ LOCAL_ASSERT(dep != NULL); -+ -+ return (const struct kbase_jd_atom *)(dep->atom); -+} -+ -+/** -+ * @brief The function retrieves a read-only reference to the dependency type field from -+ * the kbase_jd_atom_dependency structure -+ * -+ * @param[in] dep kbase jd atom dependency. -+ * -+ * @return A dependency type value. -+ */ -+static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) -+{ -+ LOCAL_ASSERT(dep != NULL); -+ -+ return dep->dep_type; -+} -+ -+/** -+ * @brief Setter macro for dep_atom array entry in kbase_jd_atom -+ * -+ * @param[in] dep The kbase jd atom dependency. -+ * @param[in] a The ATOM to be set as a dependency. -+ * @param type The ATOM dependency type to be set. -+ * -+ */ -+static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, -+ struct kbase_jd_atom *a, u8 type) -+{ -+ struct kbase_jd_atom_dependency *dep; -+ -+ LOCAL_ASSERT(const_dep != NULL); -+ -+ dep = (struct kbase_jd_atom_dependency *)const_dep; -+ -+ dep->atom = a; -+ dep->dep_type = type; -+} -+ -+/** -+ * @brief Setter macro for dep_atom array entry in kbase_jd_atom -+ * -+ * @param[in] dep The kbase jd atom dependency to be cleared. -+ * -+ */ -+static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) -+{ -+ struct kbase_jd_atom_dependency *dep; -+ -+ LOCAL_ASSERT(const_dep != NULL); -+ -+ dep = (struct kbase_jd_atom_dependency *)const_dep; -+ -+ dep->atom = NULL; -+ dep->dep_type = BASE_JD_DEP_TYPE_INVALID; -+} -+ -+enum kbase_atom_gpu_rb_state { -+ /* Atom is not currently present in slot ringbuffer */ -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, -+ /* Atom is in slot ringbuffer but is blocked on a previous atom */ -+ KBASE_ATOM_GPU_RB_WAITING_BLOCKED, -+ /* Atom is in slot ringbuffer but is waiting for a previous protected -+ * mode transition to complete */ -+ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, -+ /* Atom is in slot ringbuffer but is waiting for proected mode -+ * transition */ -+ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, -+ /* Atom is in slot ringbuffer but is waiting for cores to become -+ * available */ -+ KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, -+ /* Atom is in slot ringbuffer but is blocked on affinity */ -+ KBASE_ATOM_GPU_RB_WAITING_AFFINITY, -+ /* Atom is in slot ringbuffer and ready to run */ -+ KBASE_ATOM_GPU_RB_READY, -+ /* Atom is in slot ringbuffer and has been submitted to the GPU */ -+ KBASE_ATOM_GPU_RB_SUBMITTED, -+ /* Atom must be returned to JS as soon as it reaches the head of the -+ * ringbuffer due to a previous failure */ -+ KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 -+}; -+ -+enum kbase_atom_enter_protected_state { -+ /* -+ * Starting state: -+ * Check if a transition into protected mode is required. -+ * -+ * NOTE: The integer value of this must -+ * match KBASE_ATOM_EXIT_PROTECTED_CHECK. -+ */ -+ KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, -+ /* Wait for vinstr to suspend. */ -+ KBASE_ATOM_ENTER_PROTECTED_VINSTR, -+ /* Wait for the L2 to become idle in preparation for -+ * the coherency change. */ -+ KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, -+ /* End state; -+ * Prepare coherency change. */ -+ KBASE_ATOM_ENTER_PROTECTED_FINISHED, -+}; -+ -+enum kbase_atom_exit_protected_state { -+ /* -+ * Starting state: -+ * Check if a transition out of protected mode is required. -+ * -+ * NOTE: The integer value of this must -+ * match KBASE_ATOM_ENTER_PROTECTED_CHECK. -+ */ -+ KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, -+ /* Wait for the L2 to become idle in preparation -+ * for the reset. */ -+ KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, -+ /* Issue the protected reset. */ -+ KBASE_ATOM_EXIT_PROTECTED_RESET, -+ /* End state; -+ * Wait for the reset to complete. */ -+ KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, -+}; -+ -+struct kbase_ext_res { -+ u64 gpu_address; -+ struct kbase_mem_phy_alloc *alloc; -+}; -+ -+struct kbase_jd_atom { -+ struct work_struct work; -+ ktime_t start_timestamp; -+ -+ struct base_jd_udata udata; -+ struct kbase_context *kctx; -+ -+ struct list_head dep_head[2]; -+ struct list_head dep_item[2]; -+ const struct kbase_jd_atom_dependency dep[2]; -+ /* List head used during job dispatch job_done processing - as -+ * dependencies may not be entirely resolved at this point, we need to -+ * use a separate list head. */ -+ struct list_head jd_item; -+ /* true if atom's jd_item is currently on a list. Prevents atom being -+ * processed twice. */ -+ bool in_jd_list; -+ -+ u16 nr_extres; -+ struct kbase_ext_res *extres; -+ -+ u32 device_nr; -+ u64 affinity; -+ u64 jc; -+ enum kbase_atom_coreref_state coreref_state; -+#ifdef CONFIG_KDS -+ struct list_head node; -+ struct kds_resource_set *kds_rset; -+ bool kds_dep_satisfied; -+#endif /* CONFIG_KDS */ -+#if defined(CONFIG_SYNC) -+ /* Stores either an input or output fence, depending on soft-job type */ -+ struct sync_fence *fence; -+ struct sync_fence_waiter sync_waiter; -+#endif /* CONFIG_SYNC */ -+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) -+ struct { -+ /* Use the functions/API defined in mali_kbase_fence.h to -+ * when working with this sub struct */ -+#if defined(CONFIG_SYNC_FILE) -+ /* Input fence */ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence_in; -+#else -+ struct dma_fence *fence_in; -+#endif -+#endif -+ /* This points to the dma-buf output fence for this atom. If -+ * this is NULL then there is no fence for this atom and the -+ * following fields related to dma_fence may have invalid data. -+ * -+ * The context and seqno fields contain the details for this -+ * fence. -+ * -+ * This fence is signaled when the katom is completed, -+ * regardless of the event_code of the katom (signal also on -+ * failure). -+ */ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ /* The dma-buf fence context number for this atom. A unique -+ * context number is allocated to each katom in the context on -+ * context creation. -+ */ -+ unsigned int context; -+ /* The dma-buf fence sequence number for this atom. This is -+ * increased every time this katom uses dma-buf fence. -+ */ -+ atomic_t seqno; -+ /* This contains a list of all callbacks set up to wait on -+ * other fences. This atom must be held back from JS until all -+ * these callbacks have been called and dep_count have reached -+ * 0. The initial value of dep_count must be equal to the -+ * number of callbacks on this list. -+ * -+ * This list is protected by jctx.lock. Callbacks are added to -+ * this list when the atom is built and the wait are set up. -+ * All the callbacks then stay on the list until all callbacks -+ * have been called and the atom is queued, or cancelled, and -+ * then all callbacks are taken off the list and freed. -+ */ -+ struct list_head callbacks; -+ /* Atomic counter of number of outstandind dma-buf fence -+ * dependencies for this atom. When dep_count reaches 0 the -+ * atom may be queued. -+ * -+ * The special value "-1" may only be set after the count -+ * reaches 0, while holding jctx.lock. This indicates that the -+ * atom has been handled, either queued in JS or cancelled. -+ * -+ * If anyone but the dma-fence worker sets this to -1 they must -+ * ensure that any potentially queued worker must have -+ * completed before allowing the atom to be marked as unused. -+ * This can be done by flushing the fence work queue: -+ * kctx->dma_fence.wq. -+ */ -+ atomic_t dep_count; -+ } dma_fence; -+#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE*/ -+ -+ /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */ -+ enum base_jd_event_code event_code; -+ base_jd_core_req core_req; /**< core requirements */ -+ /** Job Slot to retry submitting to if submission from IRQ handler failed -+ * -+ * NOTE: see if this can be unified into the another member e.g. the event */ -+ int retry_submit_on_slot; -+ -+ u32 ticks; -+ /* JS atom priority with respect to other atoms on its kctx. */ -+ int sched_priority; -+ -+ int poking; /* BASE_HW_ISSUE_8316 */ -+ -+ wait_queue_head_t completed; -+ enum kbase_jd_atom_state status; -+#ifdef CONFIG_GPU_TRACEPOINTS -+ int work_id; -+#endif -+ /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */ -+ int slot_nr; -+ -+ u32 atom_flags; -+ -+ /* Number of times this atom has been retried. Used by replay soft job. -+ */ -+ int retry_count; -+ -+ enum kbase_atom_gpu_rb_state gpu_rb_state; -+ -+ u64 need_cache_flush_cores_retained; -+ -+ atomic_t blocked; -+ -+ /* Pointer to atom that this atom has same-slot dependency on */ -+ struct kbase_jd_atom *pre_dep; -+ /* Pointer to atom that has same-slot dependency on this atom */ -+ struct kbase_jd_atom *post_dep; -+ -+ /* Pointer to atom that this atom has cross-slot dependency on */ -+ struct kbase_jd_atom *x_pre_dep; -+ /* Pointer to atom that has cross-slot dependency on this atom */ -+ struct kbase_jd_atom *x_post_dep; -+ -+ /* The GPU's flush count recorded at the time of submission, used for -+ * the cache flush optimisation */ -+ u32 flush_id; -+ -+ struct kbase_jd_atom_backend backend; -+#ifdef CONFIG_DEBUG_FS -+ struct base_job_fault_event fault_event; -+#endif -+ -+ /* List head used for three different purposes: -+ * 1. Overflow list for JS ring buffers. If an atom is ready to run, -+ * but there is no room in the JS ring buffer, then the atom is put -+ * on the ring buffer's overflow list using this list node. -+ * 2. List of waiting soft jobs. -+ */ -+ struct list_head queue; -+ -+ /* Used to keep track of all JIT free/alloc jobs in submission order -+ */ -+ struct list_head jit_node; -+ bool jit_blocked; -+ -+ /* If non-zero, this indicates that the atom will fail with the set -+ * event_code when the atom is processed. */ -+ enum base_jd_event_code will_fail_event_code; -+ -+ /* Atoms will only ever be transitioning into, or out of -+ * protected mode so we do not need two separate fields. -+ */ -+ union { -+ enum kbase_atom_enter_protected_state enter; -+ enum kbase_atom_exit_protected_state exit; -+ } protected_state; -+ -+ struct rb_node runnable_tree_node; -+ -+ /* 'Age' of atom relative to other atoms in the context. */ -+ u32 age; -+}; -+ -+static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) -+{ -+ return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); -+} -+ -+/* -+ * Theory of operations: -+ * -+ * Atom objects are statically allocated within the context structure. -+ * -+ * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set. -+ */ -+ -+#define KBASE_JD_DEP_QUEUE_SIZE 256 -+ -+struct kbase_jd_context { -+ struct mutex lock; -+ struct kbasep_js_kctx_info sched_info; -+ struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; -+ -+ /** Tracks all job-dispatch jobs. This includes those not tracked by -+ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ -+ u32 job_nr; -+ -+ /** Waitq that reflects whether there are no jobs (including SW-only -+ * dependency jobs). This is set when no jobs are present on the ctx, -+ * and clear when there are jobs. -+ * -+ * @note: Job Dispatcher knows about more jobs than the Job Scheduler: -+ * the Job Scheduler is unaware of jobs that are blocked on dependencies, -+ * and SW-only dependency jobs. -+ * -+ * This waitq can be waited upon to find out when the context jobs are all -+ * done/cancelled (including those that might've been blocked on -+ * dependencies) - and so, whether it can be terminated. However, it should -+ * only be terminated once it is not present in the run-pool (see -+ * kbasep_js_kctx_info::ctx::is_scheduled). -+ * -+ * Since the waitq is only set under kbase_jd_context::lock, -+ * the waiter should also briefly obtain and drop kbase_jd_context::lock to -+ * guarentee that the setter has completed its work on the kbase_context -+ * -+ * This must be updated atomically with: -+ * - kbase_jd_context::job_nr */ -+ wait_queue_head_t zero_jobs_wait; -+ -+ /** Job Done workqueue. */ -+ struct workqueue_struct *job_done_wq; -+ -+ spinlock_t tb_lock; -+ u32 *tb; -+ size_t tb_wrap_offset; -+ -+#ifdef CONFIG_KDS -+ struct kds_callback kds_cb; -+#endif /* CONFIG_KDS */ -+#ifdef CONFIG_GPU_TRACEPOINTS -+ atomic_t work_id; -+#endif -+}; -+ -+struct kbase_device_info { -+ u32 features; -+}; -+ -+/** Poking state for BASE_HW_ISSUE_8316 */ -+enum { -+ KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0, -+ KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1 -+}; -+ -+/** Poking state for BASE_HW_ISSUE_8316 */ -+typedef u32 kbase_as_poke_state; -+ -+struct kbase_mmu_setup { -+ u64 transtab; -+ u64 memattr; -+ u64 transcfg; -+}; -+ -+/** -+ * Important: Our code makes assumptions that a struct kbase_as structure is always at -+ * kbase_device->as[number]. This is used to recover the containing -+ * struct kbase_device from a struct kbase_as structure. -+ * -+ * Therefore, struct kbase_as structures must not be allocated anywhere else. -+ */ -+struct kbase_as { -+ int number; -+ -+ struct workqueue_struct *pf_wq; -+ struct work_struct work_pagefault; -+ struct work_struct work_busfault; -+ enum kbase_mmu_fault_type fault_type; -+ bool protected_mode; -+ u32 fault_status; -+ u64 fault_addr; -+ u64 fault_extra_addr; -+ -+ struct kbase_mmu_setup current_setup; -+ -+ /* BASE_HW_ISSUE_8316 */ -+ struct workqueue_struct *poke_wq; -+ struct work_struct poke_work; -+ /** Protected by hwaccess_lock */ -+ int poke_refcount; -+ /** Protected by hwaccess_lock */ -+ kbase_as_poke_state poke_state; -+ struct hrtimer poke_timer; -+}; -+ -+static inline int kbase_as_has_bus_fault(struct kbase_as *as) -+{ -+ return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS; -+} -+ -+static inline int kbase_as_has_page_fault(struct kbase_as *as) -+{ -+ return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE; -+} -+ -+struct kbasep_mem_device { -+ atomic_t used_pages; /* Tracks usage of OS shared memory. Updated -+ when OS memory is allocated/freed. */ -+ -+}; -+ -+#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X -+ -+enum kbase_trace_code { -+ /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE -+ * THIS MUST BE USED AT THE START OF THE ENUM */ -+#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X) -+#include "mali_kbase_trace_defs.h" -+#undef KBASE_TRACE_CODE_MAKE_CODE -+ /* Comma on its own, to extend the list */ -+ , -+ /* Must be the last in the enum */ -+ KBASE_TRACE_CODE_COUNT -+}; -+ -+#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0) -+#define KBASE_TRACE_FLAG_JOBSLOT (((u8)1) << 1) -+ -+struct kbase_trace { -+ struct timespec64 timestamp; -+ u32 thread_id; -+ u32 cpu; -+ void *ctx; -+ bool katom; -+ int atom_number; -+ u64 atom_udata[2]; -+ u64 gpu_addr; -+ unsigned long info_val; -+ u8 code; -+ u8 jobslot; -+ u8 refcount; -+ u8 flags; -+}; -+ -+/** Event IDs for the power management framework. -+ * -+ * Any of these events might be missed, so they should not be relied upon to -+ * find the precise state of the GPU at a particular time in the -+ * trace. Overall, we should get a high percentage of these events for -+ * statisical purposes, and so a few missing should not be a problem */ -+enum kbase_timeline_pm_event { -+ /* helper for tests */ -+ KBASEP_TIMELINE_PM_EVENT_FIRST, -+ -+ /** Event reserved for backwards compatibility with 'init' events */ -+ KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST, -+ -+ /** The power state of the device has changed. -+ * -+ * Specifically, the device has reached a desired or available state. -+ */ -+ KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED, -+ -+ /** The GPU is becoming active. -+ * -+ * This event is sent when the first context is about to use the GPU. -+ */ -+ KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE, -+ -+ /** The GPU is becoming idle. -+ * -+ * This event is sent when the last context has finished using the GPU. -+ */ -+ KBASE_TIMELINE_PM_EVENT_GPU_IDLE, -+ -+ /** Event reserved for backwards compatibility with 'policy_change' -+ * events */ -+ KBASE_TIMELINE_PM_EVENT_RESERVED_4, -+ -+ /** Event reserved for backwards compatibility with 'system_suspend' -+ * events */ -+ KBASE_TIMELINE_PM_EVENT_RESERVED_5, -+ -+ /** Event reserved for backwards compatibility with 'system_resume' -+ * events */ -+ KBASE_TIMELINE_PM_EVENT_RESERVED_6, -+ -+ /** The job scheduler is requesting to power up/down cores. -+ * -+ * This event is sent when: -+ * - powered down cores are needed to complete a job -+ * - powered up cores are not needed anymore -+ */ -+ KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, -+ -+ KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE, -+}; -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+struct kbase_trace_kctx_timeline { -+ atomic_t jd_atoms_in_flight; -+ u32 owner_tgid; -+}; -+ -+struct kbase_trace_kbdev_timeline { -+ /* Note: strictly speaking, not needed, because it's in sync with -+ * kbase_device::jm_slots[]::submitted_nr -+ * -+ * But it's kept as an example of how to add global timeline tracking -+ * information -+ * -+ * The caller must hold hwaccess_lock when accessing this */ -+ u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS]; -+ -+ /* Last UID for each PM event */ -+ atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1]; -+ /* Counter for generating PM event UIDs */ -+ atomic_t pm_event_uid_counter; -+ /* -+ * L2 transition state - true indicates that the transition is ongoing -+ * Expected to be protected by hwaccess_lock */ -+ bool l2_transitioning; -+}; -+#endif /* CONFIG_MALI_TRACE_TIMELINE */ -+ -+ -+struct kbasep_kctx_list_element { -+ struct list_head link; -+ struct kbase_context *kctx; -+}; -+ -+/** -+ * Data stored per device for power management. -+ * -+ * This structure contains data for the power management framework. There is one -+ * instance of this structure per device in the system. -+ */ -+struct kbase_pm_device_data { -+ /** -+ * The lock protecting Power Management structures accessed outside of -+ * IRQ. -+ * -+ * This lock must also be held whenever the GPU is being powered on or -+ * off. -+ */ -+ struct mutex lock; -+ -+ /** The reference count of active contexts on this device. */ -+ int active_count; -+ /** Flag indicating suspending/suspended */ -+ bool suspending; -+ /* Wait queue set when active_count == 0 */ -+ wait_queue_head_t zero_active_count_wait; -+ -+ /** -+ * Bit masks identifying the available shader cores that are specified -+ * via sysfs. One mask per job slot. -+ */ -+ u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; -+ u64 debug_core_mask_all; -+ -+ /** -+ * Callback for initializing the runtime power management. -+ * -+ * @param kbdev The kbase device -+ * -+ * @return 0 on success, else error code -+ */ -+ int (*callback_power_runtime_init)(struct kbase_device *kbdev); -+ -+ /** -+ * Callback for terminating the runtime power management. -+ * -+ * @param kbdev The kbase device -+ */ -+ void (*callback_power_runtime_term)(struct kbase_device *kbdev); -+ -+ /* Time in milliseconds between each dvfs sample */ -+ u32 dvfs_period; -+ -+ /* Period of GPU poweroff timer */ -+ ktime_t gpu_poweroff_time; -+ -+ /* Number of ticks of GPU poweroff timer before shader is powered off */ -+ int poweroff_shader_ticks; -+ -+ /* Number of ticks of GPU poweroff timer before GPU is powered off */ -+ int poweroff_gpu_ticks; -+ -+ struct kbase_pm_backend_data backend; -+}; -+ -+/** -+ * struct kbase_mem_pool - Page based memory pool for kctx/kbdev -+ * @kbdev: Kbase device where memory is used -+ * @cur_size: Number of free pages currently in the pool (may exceed @max_size -+ * in some corner cases) -+ * @max_size: Maximum number of free pages in the pool -+ * @pool_lock: Lock protecting the pool - must be held when modifying @cur_size -+ * and @page_list -+ * @page_list: List of free pages in the pool -+ * @reclaim: Shrinker for kernel reclaim of free pages -+ * @next_pool: Pointer to next pool where pages can be allocated when this pool -+ * is empty. Pages will spill over to the next pool when this pool -+ * is full. Can be NULL if there is no next pool. -+ */ -+struct kbase_mem_pool { -+ struct kbase_device *kbdev; -+ size_t cur_size; -+ size_t max_size; -+ spinlock_t pool_lock; -+ struct list_head page_list; -+ struct shrinker reclaim; -+ -+ struct kbase_mem_pool *next_pool; -+}; -+ -+/** -+ * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP -+ * frequency, and real frequency and core mask -+ * @opp_freq: Nominal OPP frequency -+ * @real_freq: Real GPU frequency -+ * @core_mask: Shader core mask -+ */ -+struct kbase_devfreq_opp { -+ u64 opp_freq; -+ u64 real_freq; -+ u64 core_mask; -+}; -+ -+#define DEVNAME_SIZE 16 -+ -+struct kbase_device { -+ s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS]; -+ -+ u32 hw_quirks_sc; -+ u32 hw_quirks_tiler; -+ u32 hw_quirks_mmu; -+ u32 hw_quirks_jm; -+ -+ struct list_head entry; -+ struct device *dev; -+ unsigned int kbase_group_error; -+ struct miscdevice mdev; -+ u64 reg_start; -+ size_t reg_size; -+ void __iomem *reg; -+ -+ struct { -+ int irq; -+ int flags; -+ } irqs[3]; -+ -+ struct clk *clock; -+#ifdef CONFIG_REGULATOR -+ struct regulator *regulator; -+#endif -+ char devname[DEVNAME_SIZE]; -+ -+#ifdef CONFIG_MALI_NO_MALI -+ void *model; -+ struct kmem_cache *irq_slab; -+ struct workqueue_struct *irq_workq; -+ atomic_t serving_job_irq; -+ atomic_t serving_gpu_irq; -+ atomic_t serving_mmu_irq; -+ spinlock_t reg_op_lock; -+#endif /* CONFIG_MALI_NO_MALI */ -+ -+ struct kbase_pm_device_data pm; -+ struct kbasep_js_device_data js_data; -+ struct kbase_mem_pool mem_pool; -+ struct kbasep_mem_device memdev; -+ struct kbase_mmu_mode const *mmu_mode; -+ -+ struct kbase_as as[BASE_MAX_NR_AS]; -+ /* The below variables (as_free and as_to_kctx) are managed by the -+ * Context Scheduler. The kbasep_js_device_data::runpool_irq::lock must -+ * be held whilst accessing these. -+ */ -+ u16 as_free; /* Bitpattern of free Address Spaces */ -+ /* Mapping from active Address Spaces to kbase_context */ -+ struct kbase_context *as_to_kctx[BASE_MAX_NR_AS]; -+ -+ -+ spinlock_t mmu_mask_change; -+ -+ struct kbase_gpu_props gpu_props; -+ -+ /** List of SW workarounds for HW issues */ -+ unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; -+ /** List of features available */ -+ unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; -+ -+ /* Bitmaps of cores that are currently in use (running jobs). -+ * These should be kept up to date by the job scheduler. -+ * -+ * pm.power_change_lock should be held when accessing these members. -+ * -+ * kbase_pm_check_transitions_nolock() should be called when bits are -+ * cleared to update the power management system and allow transitions to -+ * occur. */ -+ u64 shader_inuse_bitmap; -+ -+ /* Refcount for cores in use */ -+ u32 shader_inuse_cnt[64]; -+ -+ /* Bitmaps of cores the JS needs for jobs ready to run */ -+ u64 shader_needed_bitmap; -+ -+ /* Refcount for cores needed */ -+ u32 shader_needed_cnt[64]; -+ -+ u32 tiler_inuse_cnt; -+ -+ u32 tiler_needed_cnt; -+ -+ /* struct for keeping track of the disjoint information -+ * -+ * The state is > 0 if the GPU is in a disjoint state. Otherwise 0 -+ * The count is the number of disjoint events that have occurred on the GPU -+ */ -+ struct { -+ atomic_t count; -+ atomic_t state; -+ } disjoint_event; -+ -+ /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */ -+ u32 l2_users_count; -+ -+ /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be -+ * submitted to these cores. These are updated by the power management code. The job scheduler should avoid -+ * submitting new jobs to any cores that are not marked as available. -+ * -+ * pm.power_change_lock should be held when accessing these members. -+ */ -+ u64 shader_available_bitmap; -+ u64 tiler_available_bitmap; -+ u64 l2_available_bitmap; -+ u64 stack_available_bitmap; -+ -+ u64 shader_ready_bitmap; -+ u64 shader_transitioning_bitmap; -+ -+ s8 nr_hw_address_spaces; /**< Number of address spaces in the GPU (constant after driver initialisation) */ -+ s8 nr_user_address_spaces; /**< Number of address spaces available to user contexts */ -+ -+ /* Structure used for instrumentation and HW counters dumping */ -+ struct kbase_hwcnt { -+ /* The lock should be used when accessing any of the following members */ -+ spinlock_t lock; -+ -+ struct kbase_context *kctx; -+ u64 addr; -+ -+ struct kbase_instr_backend backend; -+ } hwcnt; -+ -+ struct kbase_vinstr_context *vinstr_ctx; -+ -+#if KBASE_TRACE_ENABLE -+ spinlock_t trace_lock; -+ u16 trace_first_out; -+ u16 trace_next_in; -+ struct kbase_trace *trace_rbuf; -+#endif -+ -+ u32 reset_timeout_ms; -+ -+ struct mutex cacheclean_lock; -+ -+ /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */ -+ void *platform_context; -+ -+ /* List of kbase_contexts created */ -+ struct list_head kctx_list; -+ struct mutex kctx_list_lock; -+ -+ struct rockchip_opp_info opp_info; -+#ifdef CONFIG_MALI_DEVFREQ -+ struct devfreq_dev_profile devfreq_profile; -+ struct devfreq *devfreq; -+ unsigned long current_freq; -+ unsigned long current_nominal_freq; -+ unsigned long current_voltage; -+ u64 current_core_mask; -+ struct kbase_devfreq_opp *opp_table; -+ int num_opps; -+ struct monitor_dev_info *mdev_info; -+#ifdef CONFIG_DEVFREQ_THERMAL -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -+ struct devfreq_cooling_device *devfreq_cooling; -+#else -+ struct thermal_cooling_device *devfreq_cooling; -+#endif -+ /* Current IPA model - true for configured model, false for fallback */ -+ atomic_t ipa_use_configured_model; -+ struct { -+ /* Access to this struct must be with ipa.lock held */ -+ struct mutex lock; -+ struct kbase_ipa_model *configured_model; -+ struct kbase_ipa_model *fallback_model; -+ } ipa; -+#endif /* CONFIG_DEVFREQ_THERMAL */ -+#endif /* CONFIG_MALI_DEVFREQ */ -+ -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ struct kbase_trace_kbdev_timeline timeline; -+#endif -+ -+ /* -+ * Control for enabling job dump on failure, set when control debugfs -+ * is opened. -+ */ -+ bool job_fault_debug; -+ -+#ifdef CONFIG_DEBUG_FS -+ /* directory for debugfs entries */ -+ struct dentry *mali_debugfs_directory; -+ /* Root directory for per context entry */ -+ struct dentry *debugfs_ctx_directory; -+ -+#ifdef CONFIG_MALI_DEBUG -+ /* bit for each as, set if there is new data to report */ -+ u64 debugfs_as_read_bitmap; -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ /* failed job dump, used for separate debug process */ -+ wait_queue_head_t job_fault_wq; -+ wait_queue_head_t job_fault_resume_wq; -+ struct workqueue_struct *job_fault_resume_workq; -+ struct list_head job_fault_event_list; -+ spinlock_t job_fault_event_lock; -+ struct kbase_context *kctx_fault; -+ -+#if !MALI_CUSTOMER_RELEASE -+ /* Per-device data for register dumping interface */ -+ struct { -+ u16 reg_offset; /* Offset of a GPU_CONTROL register to be -+ dumped upon request */ -+ } regs_dump_debugfs_data; -+#endif /* !MALI_CUSTOMER_RELEASE */ -+#endif /* CONFIG_DEBUG_FS */ -+ -+ /* fbdump profiling controls set by gator */ -+ u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX]; -+ -+ -+#if MALI_CUSTOMER_RELEASE == 0 -+ /* Number of jobs that are run before a job is forced to fail and -+ * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced -+ * failures. */ -+ int force_replay_limit; -+ /* Count of jobs between forced failures. Incremented on each job. A -+ * job is forced to fail once this is greater than or equal to -+ * force_replay_limit. */ -+ int force_replay_count; -+ /* Core requirement for jobs to be failed and replayed. May be zero. */ -+ base_jd_core_req force_replay_core_req; -+ /* true if force_replay_limit should be randomized. The random -+ * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT. -+ */ -+ bool force_replay_random; -+#endif -+ -+ /* Total number of created contexts */ -+ atomic_t ctx_num; -+ -+#ifdef CONFIG_DEBUG_FS -+ /* Holds the most recent register accesses */ -+ struct kbase_io_history io_history; -+#endif /* CONFIG_DEBUG_FS */ -+ -+ struct kbase_hwaccess_data hwaccess; -+ -+ /* Count of page/bus faults waiting for workqueues to process */ -+ atomic_t faults_pending; -+ -+ /* true if GPU is powered off or power off operation is in progress */ -+ bool poweroff_pending; -+ -+ -+ /* defaults for new context created for this device */ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) -+ bool infinite_cache_active_default; -+#else -+ u32 infinite_cache_active_default; -+#endif -+ size_t mem_pool_max_size_default; -+ -+ /* current gpu coherency mode */ -+ u32 current_gpu_coherency_mode; -+ /* system coherency mode */ -+ u32 system_coherency; -+ /* Flag to track when cci snoops have been enabled on the interface */ -+ bool cci_snoop_enabled; -+ -+ /* SMC function IDs to call into Trusted firmware to enable/disable -+ * cache snooping. Value of 0 indicates that they are not used -+ */ -+ u32 snoop_enable_smc; -+ u32 snoop_disable_smc; -+ -+ /* Protected mode operations */ -+ struct protected_mode_ops *protected_ops; -+ -+ /* Protected device attached to this kbase device */ -+ struct protected_mode_device *protected_dev; -+ -+ /* -+ * true when GPU is put into protected mode -+ */ -+ bool protected_mode; -+ -+ /* -+ * true when GPU is transitioning into or out of protected mode -+ */ -+ bool protected_mode_transition; -+ -+ /* -+ * true if protected mode is supported -+ */ -+ bool protected_mode_support; -+ -+ -+#ifdef CONFIG_MALI_DEBUG -+ wait_queue_head_t driver_inactive_wait; -+ bool driver_inactive; -+#endif /* CONFIG_MALI_DEBUG */ -+ -+#ifdef CONFIG_MALI_FPGA_BUS_LOGGER -+ /* -+ * Bus logger integration. -+ */ -+ struct bus_logger_client *buslogger; -+#endif -+ /* Boolean indicating if an IRQ flush during reset is in progress. */ -+ bool irq_reset_flush; -+ -+ /* list of inited sub systems. Used during terminate/error recovery */ -+ u32 inited_subsys; -+ -+ spinlock_t hwaccess_lock; -+ -+ /* Protects access to MMU operations */ -+ struct mutex mmu_hw_mutex; -+ -+ /* Current serialization mode. See KBASE_SERIALIZE_* for details */ -+ u8 serialize_jobs; -+}; -+ -+/** -+ * struct jsctx_queue - JS context atom queue -+ * @runnable_tree: Root of RB-tree containing currently runnable atoms on this -+ * job slot. -+ * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot -+ * dependencies. Atoms on this list will be moved to the -+ * runnable_tree when the blocking atom completes. -+ * -+ * hwaccess_lock must be held when accessing this structure. -+ */ -+struct jsctx_queue { -+ struct rb_root runnable_tree; -+ struct list_head x_dep_head; -+}; -+ -+ -+#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ -+ (((minor) & 0xFFF) << 8) | \ -+ ((0 & 0xFF) << 0)) -+ -+/** -+ * enum kbase_context_flags - Flags for kbase contexts -+ * -+ * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit -+ * process on a 64-bit kernel. -+ * -+ * @KCTX_RUNNABLE_REF: Set when context is counted in -+ * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. -+ * -+ * @KCTX_ACTIVE: Set when the context is active. -+ * -+ * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this -+ * context. -+ * -+ * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been -+ * initialized. -+ * -+ * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new -+ * allocations. Existing allocations will not change. -+ * -+ * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. -+ * -+ * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept -+ * scheduled in. -+ * -+ * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. -+ * This is only ever updated whilst the jsctx_mutex is held. -+ * -+ * @KCTX_DYING: Set when the context process is in the process of being evicted. -+ * -+ * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this -+ * context, to disable use of implicit dma-buf fences. This is used to avoid -+ * potential synchronization deadlocks. -+ * -+ * All members need to be separate bits. This enum is intended for use in a -+ * bitmask where multiple values get OR-ed together. -+ */ -+enum kbase_context_flags { -+ KCTX_COMPAT = 1U << 0, -+ KCTX_RUNNABLE_REF = 1U << 1, -+ KCTX_ACTIVE = 1U << 2, -+ KCTX_PULLED = 1U << 3, -+ KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, -+ KCTX_INFINITE_CACHE = 1U << 5, -+ KCTX_SUBMIT_DISABLED = 1U << 6, -+ KCTX_PRIVILEGED = 1U << 7, -+ KCTX_SCHEDULED = 1U << 8, -+ KCTX_DYING = 1U << 9, -+ KCTX_NO_IMPLICIT_SYNC = 1U << 10, -+}; -+ -+struct kbase_context { -+ struct file *filp; -+ struct kbase_device *kbdev; -+ int id; /* System wide unique id */ -+ unsigned long api_version; -+ phys_addr_t pgd; -+ struct list_head event_list; -+ struct list_head event_coalesce_list; -+ struct mutex event_mutex; -+ atomic_t event_closed; -+ struct workqueue_struct *event_workq; -+ atomic_t event_count; -+ int event_coalesce_count; -+ -+ atomic_t flags; -+ -+ atomic_t setup_complete; -+ atomic_t setup_in_progress; -+ -+ u64 *mmu_teardown_pages; -+ -+ struct page *aliasing_sink_page; -+ -+ struct mutex mmu_lock; -+ struct mutex reg_lock; /* To be converted to a rwlock? */ -+ struct rb_root reg_rbtree_same; /* RB tree of GPU (live) regions, -+ * SAME_VA zone */ -+ struct rb_root reg_rbtree_exec; /* RB tree of GPU (live) regions, -+ * EXEC zone */ -+ struct rb_root reg_rbtree_custom; /* RB tree of GPU (live) regions, -+ * CUSTOM_VA zone */ -+ -+ unsigned long cookies; -+ struct kbase_va_region *pending_regions[BITS_PER_LONG]; -+ -+ wait_queue_head_t event_queue; -+ pid_t tgid; -+ pid_t pid; -+ -+ struct kbase_jd_context jctx; -+ atomic_t used_pages; -+ atomic_t nonmapped_pages; -+ -+ struct kbase_mem_pool mem_pool; -+ -+ struct shrinker reclaim; -+ struct list_head evict_list; -+ -+ struct list_head waiting_soft_jobs; -+ spinlock_t waiting_soft_jobs_lock; -+#ifdef CONFIG_KDS -+ struct list_head waiting_kds_resource; -+#endif -+#ifdef CONFIG_MALI_DMA_FENCE -+ struct { -+ struct list_head waiting_resource; -+ struct workqueue_struct *wq; -+ } dma_fence; -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ /** This is effectively part of the Run Pool, because it only has a valid -+ * setting (!=KBASEP_AS_NR_INVALID) whilst the context is scheduled in -+ * -+ * The hwaccess_lock must be held whilst accessing this. -+ * -+ * If the context relating to this as_nr is required, you must use -+ * kbasep_js_runpool_retain_ctx() to ensure that the context doesn't disappear -+ * whilst you're using it. Alternatively, just hold the hwaccess_lock -+ * to ensure the context doesn't disappear (but this has restrictions on what other locks -+ * you can take whilst doing this) */ -+ int as_nr; -+ -+ /* Keeps track of the number of users of this context. A user can be a -+ * job that is available for execution, instrumentation needing to 'pin' -+ * a context for counter collection, etc. If the refcount reaches 0 then -+ * this context is considered inactive and the previously programmed -+ * AS might be cleared at any point. -+ */ -+ atomic_t refcount; -+ -+ /* NOTE: -+ * -+ * Flags are in jctx.sched_info.ctx.flags -+ * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex -+ * -+ * All other flags must be added there */ -+ spinlock_t mm_update_lock; -+ struct mm_struct *process_mm; -+ /* End of the SAME_VA zone */ -+ u64 same_va_end; -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ struct kbase_trace_kctx_timeline timeline; -+#endif -+#ifdef CONFIG_DEBUG_FS -+ /* Content of mem_profile file */ -+ char *mem_profile_data; -+ /* Size of @c mem_profile_data */ -+ size_t mem_profile_size; -+ /* Mutex guarding memory profile state */ -+ struct mutex mem_profile_lock; -+ /* Memory profile directory under debugfs */ -+ struct dentry *kctx_dentry; -+ -+ /* for job fault debug */ -+ unsigned int *reg_dump; -+ atomic_t job_fault_count; -+ /* This list will keep the following atoms during the dump -+ * in the same context -+ */ -+ struct list_head job_fault_resume_event_list; -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+ struct jsctx_queue jsctx_queue -+ [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; -+ -+ /* Number of atoms currently pulled from this context */ -+ atomic_t atoms_pulled; -+ /* Number of atoms currently pulled from this context, per slot */ -+ atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS]; -+ /* Number of atoms currently pulled from this context, per slot and -+ * priority. Hold hwaccess_lock when accessing */ -+ int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][ -+ KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -+ -+ /* true if slot is blocked on the given priority. This will be set on a -+ * soft-stop */ -+ bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -+ -+ /* Bitmask of slots that can be pulled from */ -+ u32 slots_pullable; -+ -+ /* Backend specific data */ -+ struct kbase_context_backend backend; -+ -+ /* Work structure used for deferred ASID assignment */ -+ struct work_struct work; -+ -+ /* Only one userspace vinstr client per kbase context */ -+ struct kbase_vinstr_client *vinstr_cli; -+ struct mutex vinstr_cli_lock; -+ -+ /* List of completed jobs waiting for events to be posted */ -+ struct list_head completed_jobs; -+ /* Number of work items currently pending on job_done_wq */ -+ atomic_t work_count; -+ -+ /* Waiting soft-jobs will fail when this timer expires */ -+ struct timer_list soft_job_timeout; -+ -+ /* JIT allocation management */ -+ struct kbase_va_region *jit_alloc[256]; -+ struct list_head jit_active_head; -+ struct list_head jit_pool_head; -+ struct list_head jit_destroy_head; -+ struct mutex jit_evict_lock; -+ struct work_struct jit_work; -+ -+ /* A list of the JIT soft-jobs in submission order -+ * (protected by kbase_jd_context.lock) -+ */ -+ struct list_head jit_atoms_head; -+ /* A list of pending JIT alloc soft-jobs (using the 'queue' list_head) -+ * (protected by kbase_jd_context.lock) -+ */ -+ struct list_head jit_pending_alloc; -+ -+ /* External sticky resource management */ -+ struct list_head ext_res_meta_head; -+ -+ /* Used to record that a drain was requested from atomic context */ -+ atomic_t drain_pending; -+ -+ /* Current age count, used to determine age for newly submitted atoms */ -+ u32 age_count; -+}; -+ -+/** -+ * struct kbase_ctx_ext_res_meta - Structure which binds an external resource -+ * to a @kbase_context. -+ * @ext_res_node: List head for adding the metadata to a -+ * @kbase_context. -+ * @alloc: The physical memory allocation structure -+ * which is mapped. -+ * @gpu_addr: The GPU virtual address the resource is -+ * mapped to. -+ * -+ * External resources can be mapped into multiple contexts as well as the same -+ * context multiple times. -+ * As kbase_va_region itself isn't refcounted we can't attach our extra -+ * information to it as it could be removed under our feet leaving external -+ * resources pinned. -+ * This metadata structure binds a single external resource to a single -+ * context, ensuring that per context mapping is tracked separately so it can -+ * be overridden when needed and abuses by the application (freeing the resource -+ * multiple times) don't effect the refcount of the physical allocation. -+ */ -+struct kbase_ctx_ext_res_meta { -+ struct list_head ext_res_node; -+ struct kbase_mem_phy_alloc *alloc; -+ u64 gpu_addr; -+}; -+ -+enum kbase_reg_access_type { -+ REG_READ, -+ REG_WRITE -+}; -+ -+enum kbase_share_attr_bits { -+ /* (1ULL << 8) bit is reserved */ -+ SHARE_BOTH_BITS = (2ULL << 8), /* inner and outer shareable coherency */ -+ SHARE_INNER_BITS = (3ULL << 8) /* inner shareable coherency */ -+}; -+ -+/** -+ * kbase_device_is_cpu_coherent - Returns if the device is CPU coherent. -+ * @kbdev: kbase device -+ * -+ * Return: true if the device access are coherent, false if not. -+ */ -+static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) -+{ -+ if ((kbdev->system_coherency == COHERENCY_ACE_LITE) || -+ (kbdev->system_coherency == COHERENCY_ACE)) -+ return true; -+ -+ return false; -+} -+ -+/* Conversion helpers for setting up high resolution timers */ -+#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U)) -+#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) -+ -+/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ -+#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 -+/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ -+#define KBASE_AS_INACTIVE_MAX_LOOPS 100000 -+ -+/* Maximum number of times a job can be replayed */ -+#define BASEP_JD_REPLAY_LIMIT 15 -+ -+/* JobDescriptorHeader - taken from the architecture specifications, the layout -+ * is currently identical for all GPU archs. */ -+struct job_descriptor_header { -+ u32 exception_status; -+ u32 first_incomplete_task; -+ u64 fault_pointer; -+ u8 job_descriptor_size : 1; -+ u8 job_type : 7; -+ u8 job_barrier : 1; -+ u8 _reserved_01 : 1; -+ u8 _reserved_1 : 1; -+ u8 _reserved_02 : 1; -+ u8 _reserved_03 : 1; -+ u8 _reserved_2 : 1; -+ u8 _reserved_04 : 1; -+ u8 _reserved_05 : 1; -+ u16 job_index; -+ u16 job_dependency_index_1; -+ u16 job_dependency_index_2; -+ union { -+ u64 _64; -+ u32 _32; -+ } next_job; -+}; -+ -+#endif /* _KBASE_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_device.c b/drivers/gpu/arm/midgard/mali_kbase_device.c -new file mode 100755 -index 000000000..b0eb67da8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_device.c -@@ -0,0 +1,674 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Base kernel device APIs -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+/* NOTE: Magic - 0x45435254 (TRCE in ASCII). -+ * Supports tracing feature provided in the base module. -+ * Please keep it in sync with the value of base module. -+ */ -+#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254 -+ -+#if KBASE_TRACE_ENABLE -+static const char *kbasep_trace_code_string[] = { -+ /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE -+ * THIS MUST BE USED AT THE START OF THE ARRAY */ -+#define KBASE_TRACE_CODE_MAKE_CODE(X) # X -+#include "mali_kbase_trace_defs.h" -+#undef KBASE_TRACE_CODE_MAKE_CODE -+}; -+#endif -+ -+#define DEBUG_MESSAGE_SIZE 256 -+ -+static int kbasep_trace_init(struct kbase_device *kbdev); -+static void kbasep_trace_term(struct kbase_device *kbdev); -+static void kbasep_trace_hook_wrapper(void *param); -+ -+struct kbase_device *kbase_device_alloc(void) -+{ -+ return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); -+} -+ -+static int kbase_device_as_init(struct kbase_device *kbdev, int i) -+{ -+ const char format[] = "mali_mmu%d"; -+ char name[sizeof(format)]; -+ const char poke_format[] = "mali_mmu%d_poker"; -+ char poke_name[sizeof(poke_format)]; -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) -+ snprintf(poke_name, sizeof(poke_name), poke_format, i); -+ -+ snprintf(name, sizeof(name), format, i); -+ -+ kbdev->as[i].number = i; -+ kbdev->as[i].fault_addr = 0ULL; -+ -+ kbdev->as[i].pf_wq = alloc_workqueue(name, 0, 1); -+ if (!kbdev->as[i].pf_wq) -+ return -EINVAL; -+ -+ INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); -+ INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { -+ struct hrtimer *poke_timer = &kbdev->as[i].poke_timer; -+ struct work_struct *poke_work = &kbdev->as[i].poke_work; -+ -+ kbdev->as[i].poke_wq = alloc_workqueue(poke_name, 0, 1); -+ if (!kbdev->as[i].poke_wq) { -+ destroy_workqueue(kbdev->as[i].pf_wq); -+ return -EINVAL; -+ } -+ KBASE_DEBUG_ASSERT(!object_is_on_stack(poke_work)); -+ INIT_WORK(poke_work, kbasep_as_do_poke); -+ -+ hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ -+ poke_timer->function = kbasep_as_poke_timer_callback; -+ -+ kbdev->as[i].poke_refcount = 0; -+ kbdev->as[i].poke_state = 0u; -+ } -+ -+ return 0; -+} -+ -+static void kbase_device_as_term(struct kbase_device *kbdev, int i) -+{ -+ destroy_workqueue(kbdev->as[i].pf_wq); -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) -+ destroy_workqueue(kbdev->as[i].poke_wq); -+} -+ -+static int kbase_device_all_as_init(struct kbase_device *kbdev) -+{ -+ int i, err; -+ -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { -+ err = kbase_device_as_init(kbdev, i); -+ if (err) -+ goto free_workqs; -+ } -+ -+ return 0; -+ -+free_workqs: -+ for (; i > 0; i--) -+ kbase_device_as_term(kbdev, i); -+ -+ return err; -+} -+ -+static void kbase_device_all_as_term(struct kbase_device *kbdev) -+{ -+ int i; -+ -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) -+ kbase_device_as_term(kbdev, i); -+} -+ -+int kbase_device_init(struct kbase_device * const kbdev) -+{ -+ int i, err; -+#ifdef CONFIG_ARM64 -+ struct device_node *np = NULL; -+#endif /* CONFIG_ARM64 */ -+ -+ spin_lock_init(&kbdev->mmu_mask_change); -+ mutex_init(&kbdev->mmu_hw_mutex); -+#ifdef CONFIG_ARM64 -+ kbdev->cci_snoop_enabled = false; -+ np = kbdev->dev->of_node; -+ if (np != NULL) { -+ if (of_property_read_u32(np, "snoop_enable_smc", -+ &kbdev->snoop_enable_smc)) -+ kbdev->snoop_enable_smc = 0; -+ if (of_property_read_u32(np, "snoop_disable_smc", -+ &kbdev->snoop_disable_smc)) -+ kbdev->snoop_disable_smc = 0; -+ /* Either both or none of the calls should be provided. */ -+ if (!((kbdev->snoop_disable_smc == 0 -+ && kbdev->snoop_enable_smc == 0) -+ || (kbdev->snoop_disable_smc != 0 -+ && kbdev->snoop_enable_smc != 0))) { -+ WARN_ON(1); -+ err = -EINVAL; -+ goto fail; -+ } -+ } -+#endif /* CONFIG_ARM64 */ -+ /* Get the list of workarounds for issues on the current HW -+ * (identified by the GPU_ID register) -+ */ -+ err = kbase_hw_set_issues_mask(kbdev); -+ if (err) -+ goto fail; -+ -+ /* Set the list of features available on the current HW -+ * (identified by the GPU_ID register) -+ */ -+ kbase_hw_set_features_mask(kbdev); -+ -+ kbase_gpuprops_set_features(kbdev); -+ -+ /* On Linux 4.0+, dma coherency is determined from device tree */ -+#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) -+ set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); -+#endif -+ -+ /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our -+ * device structure was created by device-tree -+ */ -+ if (!kbdev->dev->dma_mask) -+ kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask; -+ -+ err = dma_set_mask(kbdev->dev, -+ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); -+ if (err) -+ goto dma_set_mask_failed; -+ -+ err = dma_set_coherent_mask(kbdev->dev, -+ DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits)); -+ if (err) -+ goto dma_set_mask_failed; -+ -+ kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; -+ -+ err = kbase_device_all_as_init(kbdev); -+ if (err) -+ goto as_init_failed; -+ -+ spin_lock_init(&kbdev->hwcnt.lock); -+ -+ err = kbasep_trace_init(kbdev); -+ if (err) -+ goto term_as; -+ -+ mutex_init(&kbdev->cacheclean_lock); -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) -+ kbdev->timeline.slot_atoms_submitted[i] = 0; -+ -+ for (i = 0; i <= KBASEP_TIMELINE_PM_EVENT_LAST; ++i) -+ atomic_set(&kbdev->timeline.pm_event_uid[i], 0); -+#endif /* CONFIG_MALI_TRACE_TIMELINE */ -+ -+ /* fbdump profiling controls set to 0 - fbdump not enabled until changed by gator */ -+ for (i = 0; i < FBDUMP_CONTROL_MAX; i++) -+ kbdev->kbase_profiling_controls[i] = 0; -+ -+ kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev); -+ -+ atomic_set(&kbdev->ctx_num, 0); -+ -+ err = kbase_instr_backend_init(kbdev); -+ if (err) -+ goto term_trace; -+ -+ kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; -+ -+ kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); -+ else -+ kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); -+ -+#ifdef CONFIG_MALI_DEBUG -+ init_waitqueue_head(&kbdev->driver_inactive_wait); -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ return 0; -+term_trace: -+ kbasep_trace_term(kbdev); -+term_as: -+ kbase_device_all_as_term(kbdev); -+as_init_failed: -+dma_set_mask_failed: -+fail: -+ return err; -+} -+ -+void kbase_device_term(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+#if KBASE_TRACE_ENABLE -+ kbase_debug_assert_register_hook(NULL, NULL); -+#endif -+ -+ kbase_instr_backend_term(kbdev); -+ -+ kbasep_trace_term(kbdev); -+ -+ kbase_device_all_as_term(kbdev); -+} -+ -+void kbase_device_free(struct kbase_device *kbdev) -+{ -+ kfree(kbdev); -+} -+ -+int kbase_device_trace_buffer_install( -+ struct kbase_context *kctx, u32 *tb, size_t size) -+{ -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(tb); -+ -+ /* Interface uses 16-bit value to track last accessed entry. Each entry -+ * is composed of two 32-bit words. -+ * This limits the size that can be handled without an overflow. */ -+ if (0xFFFF * (2 * sizeof(u32)) < size) -+ return -EINVAL; -+ -+ /* set up the header */ -+ /* magic number in the first 4 bytes */ -+ tb[0] = TRACE_BUFFER_HEADER_SPECIAL; -+ /* Store (write offset = 0, wrap counter = 0, transaction active = no) -+ * write offset 0 means never written. -+ * Offsets 1 to (wrap_offset - 1) used to store values when trace started -+ */ -+ tb[1] = 0; -+ -+ /* install trace buffer */ -+ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); -+ kctx->jctx.tb_wrap_offset = size / 8; -+ kctx->jctx.tb = tb; -+ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); -+ -+ return 0; -+} -+ -+void kbase_device_trace_buffer_uninstall(struct kbase_context *kctx) -+{ -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); -+ kctx->jctx.tb = NULL; -+ kctx->jctx.tb_wrap_offset = 0; -+ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); -+} -+ -+void kbase_device_trace_register_access(struct kbase_context *kctx, enum kbase_reg_access_type type, u16 reg_offset, u32 reg_value) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kctx->jctx.tb_lock, flags); -+ if (kctx->jctx.tb) { -+ u16 wrap_count; -+ u16 write_offset; -+ u32 *tb = kctx->jctx.tb; -+ u32 header_word; -+ -+ header_word = tb[1]; -+ KBASE_DEBUG_ASSERT(0 == (header_word & 0x1)); -+ -+ wrap_count = (header_word >> 1) & 0x7FFF; -+ write_offset = (header_word >> 16) & 0xFFFF; -+ -+ /* mark as transaction in progress */ -+ tb[1] |= 0x1; -+ mb(); -+ -+ /* calculate new offset */ -+ write_offset++; -+ if (write_offset == kctx->jctx.tb_wrap_offset) { -+ /* wrap */ -+ write_offset = 1; -+ wrap_count++; -+ wrap_count &= 0x7FFF; /* 15bit wrap counter */ -+ } -+ -+ /* store the trace entry at the selected offset */ -+ tb[write_offset * 2 + 0] = (reg_offset & ~0x3) | ((type == REG_WRITE) ? 0x1 : 0x0); -+ tb[write_offset * 2 + 1] = reg_value; -+ mb(); -+ -+ /* new header word */ -+ header_word = (write_offset << 16) | (wrap_count << 1) | 0x0; /* transaction complete */ -+ tb[1] = header_word; -+ } -+ spin_unlock_irqrestore(&kctx->jctx.tb_lock, flags); -+} -+ -+/* -+ * Device trace functions -+ */ -+#if KBASE_TRACE_ENABLE -+ -+static int kbasep_trace_init(struct kbase_device *kbdev) -+{ -+ struct kbase_trace *rbuf; -+ -+ rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); -+ -+ if (!rbuf) -+ return -EINVAL; -+ -+ kbdev->trace_rbuf = rbuf; -+ spin_lock_init(&kbdev->trace_lock); -+ return 0; -+} -+ -+static void kbasep_trace_term(struct kbase_device *kbdev) -+{ -+ kfree(kbdev->trace_rbuf); -+} -+ -+static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len) -+{ -+ s32 written = 0; -+ -+ /* Initial part of message */ -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0); -+ -+ if (trace_msg->katom) -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0); -+ -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0); -+ -+ /* NOTE: Could add function callbacks to handle different message types */ -+ /* Jobslot present */ -+ if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT) -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0); -+ -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); -+ -+ /* Refcount present */ -+ if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT) -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0); -+ -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0); -+ -+ /* Rest of message */ -+ written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0); -+} -+ -+static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg) -+{ -+ char buffer[DEBUG_MESSAGE_SIZE]; -+ -+ kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); -+ dev_dbg(kbdev->dev, "%s", buffer); -+} -+ -+void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val) -+{ -+ unsigned long irqflags; -+ struct kbase_trace *trace_msg; -+ -+ spin_lock_irqsave(&kbdev->trace_lock, irqflags); -+ -+ trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in]; -+ -+ /* Fill the message */ -+ trace_msg->thread_id = task_pid_nr(current); -+ trace_msg->cpu = task_cpu(current); -+ -+ ktime_get_real_ts64(&trace_msg->timestamp); -+ -+ trace_msg->code = code; -+ trace_msg->ctx = ctx; -+ -+ if (NULL == katom) { -+ trace_msg->katom = false; -+ } else { -+ trace_msg->katom = true; -+ trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom); -+ trace_msg->atom_udata[0] = katom->udata.blob[0]; -+ trace_msg->atom_udata[1] = katom->udata.blob[1]; -+ } -+ -+ trace_msg->gpu_addr = gpu_addr; -+ trace_msg->jobslot = jobslot; -+ trace_msg->refcount = MIN((unsigned int)refcount, 0xFF); -+ trace_msg->info_val = info_val; -+ trace_msg->flags = flags; -+ -+ /* Update the ringbuffer indices */ -+ kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK; -+ if (kbdev->trace_next_in == kbdev->trace_first_out) -+ kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK; -+ -+ /* Done */ -+ -+ spin_unlock_irqrestore(&kbdev->trace_lock, irqflags); -+} -+ -+void kbasep_trace_clear(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->trace_lock, flags); -+ kbdev->trace_first_out = kbdev->trace_next_in; -+ spin_unlock_irqrestore(&kbdev->trace_lock, flags); -+} -+ -+void kbasep_trace_dump(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ u32 start; -+ u32 end; -+ -+ dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val"); -+ spin_lock_irqsave(&kbdev->trace_lock, flags); -+ start = kbdev->trace_first_out; -+ end = kbdev->trace_next_in; -+ -+ while (start != end) { -+ struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start]; -+ -+ kbasep_trace_dump_msg(kbdev, trace_msg); -+ -+ start = (start + 1) & KBASE_TRACE_MASK; -+ } -+ dev_dbg(kbdev->dev, "TRACE_END"); -+ -+ spin_unlock_irqrestore(&kbdev->trace_lock, flags); -+ -+ KBASE_TRACE_CLEAR(kbdev); -+} -+ -+static void kbasep_trace_hook_wrapper(void *param) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)param; -+ -+ kbasep_trace_dump(kbdev); -+} -+ -+#ifdef CONFIG_DEBUG_FS -+struct trace_seq_state { -+ struct kbase_trace trace_buf[KBASE_TRACE_SIZE]; -+ u32 start; -+ u32 end; -+}; -+ -+static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos) -+{ -+ struct trace_seq_state *state = s->private; -+ int i; -+ -+ if (*pos > KBASE_TRACE_SIZE) -+ return NULL; -+ i = state->start + *pos; -+ if ((state->end >= state->start && i >= state->end) || -+ i >= state->end + KBASE_TRACE_SIZE) -+ return NULL; -+ -+ i &= KBASE_TRACE_MASK; -+ -+ return &state->trace_buf[i]; -+} -+ -+static void kbasep_trace_seq_stop(struct seq_file *s, void *data) -+{ -+} -+ -+static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos) -+{ -+ struct trace_seq_state *state = s->private; -+ int i; -+ -+ (*pos)++; -+ -+ i = (state->start + *pos) & KBASE_TRACE_MASK; -+ if (i == state->end) -+ return NULL; -+ -+ return &state->trace_buf[i]; -+} -+ -+static int kbasep_trace_seq_show(struct seq_file *s, void *data) -+{ -+ struct kbase_trace *trace_msg = data; -+ char buffer[DEBUG_MESSAGE_SIZE]; -+ -+ kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE); -+ seq_printf(s, "%s\n", buffer); -+ return 0; -+} -+ -+static const struct seq_operations kbasep_trace_seq_ops = { -+ .start = kbasep_trace_seq_start, -+ .next = kbasep_trace_seq_next, -+ .stop = kbasep_trace_seq_stop, -+ .show = kbasep_trace_seq_show, -+}; -+ -+static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file) -+{ -+ struct kbase_device *kbdev = inode->i_private; -+ unsigned long flags; -+ -+ struct trace_seq_state *state; -+ -+ state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state)); -+ if (!state) -+ return -ENOMEM; -+ -+ spin_lock_irqsave(&kbdev->trace_lock, flags); -+ state->start = kbdev->trace_first_out; -+ state->end = kbdev->trace_next_in; -+ memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf)); -+ spin_unlock_irqrestore(&kbdev->trace_lock, flags); -+ -+ return 0; -+} -+ -+static const struct file_operations kbasep_trace_debugfs_fops = { -+ .open = kbasep_trace_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release_private, -+}; -+ -+void kbasep_trace_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("mali_trace", S_IRUGO, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_trace_debugfs_fops); -+} -+ -+#else -+void kbasep_trace_debugfs_init(struct kbase_device *kbdev) -+{ -+} -+#endif /* CONFIG_DEBUG_FS */ -+ -+#else /* KBASE_TRACE_ENABLE */ -+static int kbasep_trace_init(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+ return 0; -+} -+ -+static void kbasep_trace_term(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+static void kbasep_trace_hook_wrapper(void *param) -+{ -+ CSTD_UNUSED(param); -+} -+ -+void kbasep_trace_dump(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+#endif /* KBASE_TRACE_ENABLE */ -+ -+void kbase_set_profiling_control(struct kbase_device *kbdev, u32 control, u32 value) -+{ -+ switch (control) { -+ case FBDUMP_CONTROL_ENABLE: -+ /* fall through */ -+ case FBDUMP_CONTROL_RATE: -+ /* fall through */ -+ case SW_COUNTER_ENABLE: -+ /* fall through */ -+ case FBDUMP_CONTROL_RESIZE_FACTOR: -+ kbdev->kbase_profiling_controls[control] = value; -+ break; -+ default: -+ dev_err(kbdev->dev, "Profiling control %d not found\n", control); -+ break; -+ } -+} -+ -+/* -+ * Called by gator to control the production of -+ * profiling information at runtime -+ * */ -+ -+void _mali_profiling_control(u32 action, u32 value) -+{ -+ struct kbase_device *kbdev = NULL; -+ -+ /* find the first i.e. call with -1 */ -+ kbdev = kbase_find_device(-1); -+ -+ if (NULL != kbdev) -+ kbase_set_profiling_control(kbdev, action, value); -+} -+KBASE_EXPORT_SYMBOL(_mali_profiling_control); -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c b/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c -new file mode 100755 -index 000000000..f70bcccf4 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c -@@ -0,0 +1,76 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Base kernel disjoint events helper functions -+ */ -+ -+#include -+ -+void kbase_disjoint_init(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ atomic_set(&kbdev->disjoint_event.count, 0); -+ atomic_set(&kbdev->disjoint_event.state, 0); -+} -+ -+/* increment the disjoint event count */ -+void kbase_disjoint_event(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ atomic_inc(&kbdev->disjoint_event.count); -+} -+ -+/* increment the state and the event counter */ -+void kbase_disjoint_state_up(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ atomic_inc(&kbdev->disjoint_event.state); -+ -+ kbase_disjoint_event(kbdev); -+} -+ -+/* decrement the state */ -+void kbase_disjoint_state_down(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(atomic_read(&kbdev->disjoint_event.state) > 0); -+ -+ kbase_disjoint_event(kbdev); -+ -+ atomic_dec(&kbdev->disjoint_event.state); -+} -+ -+/* increments the count only if the state is > 0 */ -+void kbase_disjoint_event_potential(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ if (atomic_read(&kbdev->disjoint_event.state)) -+ kbase_disjoint_event(kbdev); -+} -+ -+u32 kbase_disjoint_event_get(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ return atomic_read(&kbdev->disjoint_event.count); -+} -+KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); -diff --git a/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c -new file mode 100755 -index 000000000..9197743c8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c -@@ -0,0 +1,449 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_DMA_FENCE as -+ * it will be set there. -+ */ -+#include "mali_kbase_dma_fence.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+static void -+kbase_dma_fence_work(struct work_struct *pwork); -+ -+static void -+kbase_dma_fence_waiters_add(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ list_add_tail(&katom->queue, &kctx->dma_fence.waiting_resource); -+} -+ -+static void -+kbase_dma_fence_waiters_remove(struct kbase_jd_atom *katom) -+{ -+ list_del(&katom->queue); -+} -+ -+static int -+kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, -+ struct ww_acquire_ctx *ctx) -+{ -+ struct reservation_object *content_res = NULL; -+ unsigned int content_res_idx = 0; -+ unsigned int r; -+ int err = 0; -+ -+ ww_acquire_init(ctx, &reservation_ww_class); -+ -+retry: -+ for (r = 0; r < info->dma_fence_resv_count; r++) { -+ if (info->resv_objs[r] == content_res) { -+ content_res = NULL; -+ continue; -+ } -+ -+ err = ww_mutex_lock(&info->resv_objs[r]->lock, ctx); -+ if (err) -+ goto error; -+ } -+ -+ ww_acquire_done(ctx); -+ return err; -+ -+error: -+ content_res_idx = r; -+ -+ /* Unlock the locked one ones */ -+ while (r--) -+ ww_mutex_unlock(&info->resv_objs[r]->lock); -+ -+ if (content_res) -+ ww_mutex_unlock(&content_res->lock); -+ -+ /* If we deadlock try with lock_slow and retry */ -+ if (err == -EDEADLK) { -+ content_res = info->resv_objs[content_res_idx]; -+ ww_mutex_lock_slow(&content_res->lock, ctx); -+ goto retry; -+ } -+ -+ /* If we are here the function failed */ -+ ww_acquire_fini(ctx); -+ return err; -+} -+ -+static void -+kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info, -+ struct ww_acquire_ctx *ctx) -+{ -+ unsigned int r; -+ -+ for (r = 0; r < info->dma_fence_resv_count; r++) -+ ww_mutex_unlock(&info->resv_objs[r]->lock); -+ ww_acquire_fini(ctx); -+} -+ -+/** -+ * kbase_dma_fence_queue_work() - Queue work to handle @katom -+ * @katom: Pointer to atom for which to queue work -+ * -+ * Queue kbase_dma_fence_work() for @katom to clean up the fence callbacks and -+ * submit the atom. -+ */ -+static void -+kbase_dma_fence_queue_work(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ bool ret; -+ -+ INIT_WORK(&katom->work, kbase_dma_fence_work); -+ ret = queue_work(kctx->dma_fence.wq, &katom->work); -+ /* Warn if work was already queued, that should not happen. */ -+ WARN_ON(!ret); -+} -+ -+/** -+ * kbase_dma_fence_cancel_atom() - Cancels waiting on an atom -+ * @katom: Katom to cancel -+ * -+ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. -+ */ -+static void -+kbase_dma_fence_cancel_atom(struct kbase_jd_atom *katom) -+{ -+ lockdep_assert_held(&katom->kctx->jctx.lock); -+ -+ /* Cancel callbacks and clean up. */ -+ kbase_fence_free_callbacks(katom); -+ -+ /* Mark the atom as handled in case all fences signaled just before -+ * canceling the callbacks and the worker was queued. -+ */ -+ kbase_fence_dep_count_set(katom, -1); -+ -+ /* Prevent job_done_nolock from being called twice on an atom when -+ * there is a race between job completion and cancellation. -+ */ -+ -+ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { -+ /* Wait was cancelled - zap the atom */ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ if (jd_done_nolock(katom, NULL)) -+ kbase_js_sched_all(katom->kctx->kbdev); -+ } -+} -+ -+/** -+ * kbase_dma_fence_work() - Worker thread called when a fence is signaled -+ * @pwork: work_struct containing a pointer to a katom -+ * -+ * This function will clean and mark all dependencies as satisfied -+ */ -+static void -+kbase_dma_fence_work(struct work_struct *pwork) -+{ -+ struct kbase_jd_atom *katom; -+ struct kbase_jd_context *ctx; -+ -+ katom = container_of(pwork, struct kbase_jd_atom, work); -+ ctx = &katom->kctx->jctx; -+ -+ mutex_lock(&ctx->lock); -+ if (kbase_fence_dep_count_read(katom) != 0) -+ goto out; -+ -+ kbase_fence_dep_count_set(katom, -1); -+ -+ /* Remove atom from list of dma-fence waiting atoms. */ -+ kbase_dma_fence_waiters_remove(katom); -+ /* Cleanup callbacks. */ -+ kbase_fence_free_callbacks(katom); -+ /* -+ * Queue atom on GPU, unless it has already completed due to a failing -+ * dependency. Run jd_done_nolock() on the katom if it is completed. -+ */ -+ if (unlikely(katom->status == KBASE_JD_ATOM_STATE_COMPLETED)) -+ jd_done_nolock(katom, NULL); -+ else -+ kbase_jd_dep_clear_locked(katom); -+ -+out: -+ mutex_unlock(&ctx->lock); -+} -+ -+static void -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) -+#else -+kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -+#endif -+{ -+ struct kbase_fence_cb *kcb = container_of(cb, -+ struct kbase_fence_cb, -+ fence_cb); -+ struct kbase_jd_atom *katom = kcb->katom; -+ -+ /* If the atom is zapped dep_count will be forced to a negative number -+ * preventing this callback from ever scheduling work. Which in turn -+ * would reschedule the atom. -+ */ -+ -+ if (kbase_fence_dep_count_dec_and_test(katom)) -+ kbase_dma_fence_queue_work(katom); -+} -+ -+static int -+kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, -+ struct reservation_object *resv, -+ bool exclusive) -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *excl_fence = NULL; -+ struct fence **shared_fences = NULL; -+#else -+ struct dma_fence *excl_fence = NULL; -+ struct dma_fence **shared_fences = NULL; -+#endif -+ unsigned int shared_count = 0; -+ int err, i; -+ -+ err = reservation_object_get_fences_rcu(resv, -+ &excl_fence, -+ &shared_count, -+ &shared_fences); -+ if (err) -+ return err; -+ -+ if (excl_fence) { -+ err = kbase_fence_add_callback(katom, -+ excl_fence, -+ kbase_dma_fence_cb); -+ -+ /* Release our reference, taken by reservation_object_get_fences_rcu(), -+ * to the fence. We have set up our callback (if that was possible), -+ * and it's the fence's owner is responsible for singling the fence -+ * before allowing it to disappear. -+ */ -+ dma_fence_put(excl_fence); -+ -+ if (err) -+ goto out; -+ } -+ -+ if (exclusive) { -+ for (i = 0; i < shared_count; i++) { -+ err = kbase_fence_add_callback(katom, -+ shared_fences[i], -+ kbase_dma_fence_cb); -+ if (err) -+ goto out; -+ } -+ } -+ -+ /* Release all our references to the shared fences, taken by -+ * reservation_object_get_fences_rcu(). We have set up our callback (if -+ * that was possible), and it's the fence's owner is responsible for -+ * signaling the fence before allowing it to disappear. -+ */ -+out: -+ for (i = 0; i < shared_count; i++) -+ dma_fence_put(shared_fences[i]); -+ kfree(shared_fences); -+ -+ if (err) { -+ /* -+ * On error, cancel and clean up all callbacks that was set up -+ * before the error. -+ */ -+ kbase_fence_free_callbacks(katom); -+ } -+ -+ return err; -+} -+ -+void kbase_dma_fence_add_reservation(struct reservation_object *resv, -+ struct kbase_dma_fence_resv_info *info, -+ bool exclusive) -+{ -+ unsigned int i; -+ -+ for (i = 0; i < info->dma_fence_resv_count; i++) { -+ /* Duplicate resource, ignore */ -+ if (info->resv_objs[i] == resv) -+ return; -+ } -+ -+ info->resv_objs[info->dma_fence_resv_count] = resv; -+ if (exclusive) -+ set_bit(info->dma_fence_resv_count, -+ info->dma_fence_excl_bitmap); -+ (info->dma_fence_resv_count)++; -+} -+ -+int kbase_dma_fence_wait(struct kbase_jd_atom *katom, -+ struct kbase_dma_fence_resv_info *info) -+{ -+ int err, i; -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ struct ww_acquire_ctx ww_ctx; -+ -+ lockdep_assert_held(&katom->kctx->jctx.lock); -+ -+ fence = kbase_fence_out_new(katom); -+ if (!fence) { -+ err = -ENOMEM; -+ dev_err(katom->kctx->kbdev->dev, -+ "Error %d creating fence.\n", err); -+ return err; -+ } -+ -+ kbase_fence_dep_count_set(katom, 1); -+ -+ err = kbase_dma_fence_lock_reservations(info, &ww_ctx); -+ if (err) { -+ dev_err(katom->kctx->kbdev->dev, -+ "Error %d locking reservations.\n", err); -+ kbase_fence_dep_count_set(katom, -1); -+ kbase_fence_out_remove(katom); -+ return err; -+ } -+ -+ for (i = 0; i < info->dma_fence_resv_count; i++) { -+ struct reservation_object *obj = info->resv_objs[i]; -+ -+ if (!test_bit(i, info->dma_fence_excl_bitmap)) { -+ err = reservation_object_reserve_shared(obj); -+ if (err) { -+ dev_err(katom->kctx->kbdev->dev, -+ "Error %d reserving space for shared fence.\n", err); -+ goto end; -+ } -+ -+ err = kbase_dma_fence_add_reservation_callback(katom, obj, false); -+ if (err) { -+ dev_err(katom->kctx->kbdev->dev, -+ "Error %d adding reservation to callback.\n", err); -+ goto end; -+ } -+ -+ reservation_object_add_shared_fence(obj, fence); -+ } else { -+ err = kbase_dma_fence_add_reservation_callback(katom, obj, true); -+ if (err) { -+ dev_err(katom->kctx->kbdev->dev, -+ "Error %d adding reservation to callback.\n", err); -+ goto end; -+ } -+ -+ reservation_object_add_excl_fence(obj, fence); -+ } -+ } -+ -+end: -+ kbase_dma_fence_unlock_reservations(info, &ww_ctx); -+ -+ if (likely(!err)) { -+ /* Test if the callbacks are already triggered */ -+ if (kbase_fence_dep_count_dec_and_test(katom)) { -+ kbase_fence_dep_count_set(katom, -1); -+ kbase_fence_free_callbacks(katom); -+ } else { -+ /* Add katom to the list of dma-buf fence waiting atoms -+ * only if it is still waiting. -+ */ -+ kbase_dma_fence_waiters_add(katom); -+ } -+ } else { -+ /* There was an error, cancel callbacks, set dep_count to -1 to -+ * indicate that the atom has been handled (the caller will -+ * kill it for us), signal the fence, free callbacks and the -+ * fence. -+ */ -+ kbase_fence_free_callbacks(katom); -+ kbase_fence_dep_count_set(katom, -1); -+ kbase_dma_fence_signal(katom); -+ } -+ -+ return err; -+} -+ -+void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx) -+{ -+ struct list_head *list = &kctx->dma_fence.waiting_resource; -+ -+ while (!list_empty(list)) { -+ struct kbase_jd_atom *katom; -+ -+ katom = list_first_entry(list, struct kbase_jd_atom, queue); -+ kbase_dma_fence_waiters_remove(katom); -+ kbase_dma_fence_cancel_atom(katom); -+ } -+} -+ -+void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom) -+{ -+ /* Cancel callbacks and clean up. */ -+ if (kbase_fence_free_callbacks(katom)) -+ kbase_dma_fence_queue_work(katom); -+} -+ -+void kbase_dma_fence_signal(struct kbase_jd_atom *katom) -+{ -+ if (!katom->dma_fence.fence) -+ return; -+ -+ /* Signal the atom's fence. */ -+ dma_fence_signal(katom->dma_fence.fence); -+ -+ kbase_fence_out_remove(katom); -+ -+ kbase_fence_free_callbacks(katom); -+} -+ -+void kbase_dma_fence_term(struct kbase_context *kctx) -+{ -+ destroy_workqueue(kctx->dma_fence.wq); -+ kctx->dma_fence.wq = NULL; -+} -+ -+int kbase_dma_fence_init(struct kbase_context *kctx) -+{ -+ INIT_LIST_HEAD(&kctx->dma_fence.waiting_resource); -+ -+ kctx->dma_fence.wq = alloc_workqueue("mali-fence-%d", -+ WQ_UNBOUND, 1, kctx->pid); -+ if (!kctx->dma_fence.wq) -+ return -ENOMEM; -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h -new file mode 100755 -index 000000000..c9ab40350 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h -@@ -0,0 +1,131 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_DMA_FENCE_H_ -+#define _KBASE_DMA_FENCE_H_ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ -+#include -+#include -+#include -+ -+ -+/* Forward declaration from mali_kbase_defs.h */ -+struct kbase_jd_atom; -+struct kbase_context; -+ -+/** -+ * struct kbase_dma_fence_resv_info - Structure with list of reservation objects -+ * @resv_objs: Array of reservation objects to attach the -+ * new fence to. -+ * @dma_fence_resv_count: Number of reservation objects in the array. -+ * @dma_fence_excl_bitmap: Specifies which resv_obj are exclusive. -+ * -+ * This is used by some functions to pass around a collection of data about -+ * reservation objects. -+ */ -+struct kbase_dma_fence_resv_info { -+ struct reservation_object **resv_objs; -+ unsigned int dma_fence_resv_count; -+ unsigned long *dma_fence_excl_bitmap; -+}; -+ -+/** -+ * kbase_dma_fence_add_reservation() - Adds a resv to the array of resv_objs -+ * @resv: Reservation object to add to the array. -+ * @info: Pointer to struct with current reservation info -+ * @exclusive: Boolean indicating if exclusive access is needed -+ * -+ * The function adds a new reservation_object to an existing array of -+ * reservation_objects. At the same time keeps track of which objects require -+ * exclusive access in dma_fence_excl_bitmap. -+ */ -+void kbase_dma_fence_add_reservation(struct reservation_object *resv, -+ struct kbase_dma_fence_resv_info *info, -+ bool exclusive); -+ -+/** -+ * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs -+ * @katom: Katom with the external dependency. -+ * @info: Pointer to struct with current reservation info -+ * -+ * Return: An error code or 0 if succeeds -+ */ -+int kbase_dma_fence_wait(struct kbase_jd_atom *katom, -+ struct kbase_dma_fence_resv_info *info); -+ -+/** -+ * kbase_dma_fence_cancel_ctx() - Cancel all dma-fences blocked atoms on kctx -+ * @kctx: Pointer to kbase context -+ * -+ * This function will cancel and clean up all katoms on @kctx that is waiting -+ * on dma-buf fences. -+ * -+ * Locking: jctx.lock needs to be held when calling this function. -+ */ -+void kbase_dma_fence_cancel_all_atoms(struct kbase_context *kctx); -+ -+/** -+ * kbase_dma_fence_cancel_callbacks() - Cancel only callbacks on katom -+ * @katom: Pointer to katom whose callbacks are to be canceled -+ * -+ * This function cancels all dma-buf fence callbacks on @katom, but does not -+ * cancel the katom itself. -+ * -+ * The caller is responsible for ensuring that jd_done_nolock is called on -+ * @katom. -+ * -+ * Locking: jctx.lock must be held when calling this function. -+ */ -+void kbase_dma_fence_cancel_callbacks(struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_dma_fence_signal() - Signal katom's fence and clean up after wait -+ * @katom: Pointer to katom to signal and clean up -+ * -+ * This function will signal the @katom's fence, if it has one, and clean up -+ * the callback data from the katom's wait on earlier fences. -+ * -+ * Locking: jctx.lock must be held while calling this function. -+ */ -+void kbase_dma_fence_signal(struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_dma_fence_term() - Terminate Mali dma-fence context -+ * @kctx: kbase context to terminate -+ */ -+void kbase_dma_fence_term(struct kbase_context *kctx); -+ -+/** -+ * kbase_dma_fence_init() - Initialize Mali dma-fence context -+ * @kctx: kbase context to initialize -+ */ -+int kbase_dma_fence_init(struct kbase_context *kctx); -+ -+ -+#else /* CONFIG_MALI_DMA_FENCE */ -+/* Dummy functions for when dma-buf fence isn't enabled. */ -+ -+static inline int kbase_dma_fence_init(struct kbase_context *kctx) -+{ -+ return 0; -+} -+ -+static inline void kbase_dma_fence_term(struct kbase_context *kctx) {} -+#endif /* CONFIG_MALI_DMA_FENCE */ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_event.c b/drivers/gpu/arm/midgard/mali_kbase_event.c -new file mode 100755 -index 000000000..188148645 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_event.c -@@ -0,0 +1,259 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+#include -+#include -+ -+static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ struct base_jd_udata data; -+ -+ lockdep_assert_held(&kctx->jctx.lock); -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(katom != NULL); -+ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); -+ -+ data = katom->udata; -+ -+ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(1, &kctx->timeline.jd_atoms_in_flight)); -+ -+ KBASE_TLSTREAM_TL_NRET_ATOM_CTX(katom, kctx); -+ KBASE_TLSTREAM_TL_DEL_ATOM(katom); -+ -+ katom->status = KBASE_JD_ATOM_STATE_UNUSED; -+ -+ wake_up(&katom->completed); -+ -+ return data; -+} -+ -+int kbase_event_pending(struct kbase_context *ctx) -+{ -+ KBASE_DEBUG_ASSERT(ctx); -+ -+ return (atomic_read(&ctx->event_count) != 0) || -+ (atomic_read(&ctx->event_closed) != 0); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_event_pending); -+ -+int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent) -+{ -+ struct kbase_jd_atom *atom; -+ -+ KBASE_DEBUG_ASSERT(ctx); -+ -+ mutex_lock(&ctx->event_mutex); -+ -+ if (list_empty(&ctx->event_list)) { -+ if (!atomic_read(&ctx->event_closed)) { -+ mutex_unlock(&ctx->event_mutex); -+ return -1; -+ } -+ -+ /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ -+ mutex_unlock(&ctx->event_mutex); -+ uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; -+ memset(&uevent->udata, 0, sizeof(uevent->udata)); -+ dev_dbg(ctx->kbdev->dev, -+ "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", -+ BASE_JD_EVENT_DRV_TERMINATED); -+ return 0; -+ } -+ -+ /* normal event processing */ -+ atomic_dec(&ctx->event_count); -+ atom = list_entry(ctx->event_list.next, struct kbase_jd_atom, dep_item[0]); -+ list_del(ctx->event_list.next); -+ -+ mutex_unlock(&ctx->event_mutex); -+ -+ dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); -+ uevent->event_code = atom->event_code; -+ uevent->atom_number = (atom - ctx->jctx.atoms); -+ -+ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) -+ kbase_jd_free_external_resources(atom); -+ -+ mutex_lock(&ctx->jctx.lock); -+ uevent->udata = kbase_event_process(ctx, atom); -+ mutex_unlock(&ctx->jctx.lock); -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_event_dequeue); -+ -+/** -+ * kbase_event_process_noreport_worker - Worker for processing atoms that do not -+ * return an event but do have external -+ * resources -+ * @data: Work structure -+ */ -+static void kbase_event_process_noreport_worker(struct work_struct *data) -+{ -+ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, -+ work); -+ struct kbase_context *kctx = katom->kctx; -+ -+ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) -+ kbase_jd_free_external_resources(katom); -+ -+ mutex_lock(&kctx->jctx.lock); -+ kbase_event_process(kctx, katom); -+ mutex_unlock(&kctx->jctx.lock); -+} -+ -+/** -+ * kbase_event_process_noreport - Process atoms that do not return an event -+ * @kctx: Context pointer -+ * @katom: Atom to be processed -+ * -+ * Atoms that do not have external resources will be processed immediately. -+ * Atoms that do have external resources will be processed on a workqueue, in -+ * order to avoid locking issues. -+ */ -+static void kbase_event_process_noreport(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom) -+{ -+ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { -+ INIT_WORK(&katom->work, kbase_event_process_noreport_worker); -+ queue_work(kctx->event_workq, &katom->work); -+ } else { -+ kbase_event_process(kctx, katom); -+ } -+} -+ -+/** -+ * kbase_event_coalesce - Move pending events to the main event list -+ * @kctx: Context pointer -+ * -+ * kctx->event_list and kctx->event_coalesce_count must be protected -+ * by a lock unless this is the last thread using them -+ * (and we're about to terminate the lock). -+ * -+ * Return: The number of pending events moved to the main event list -+ */ -+static int kbase_event_coalesce(struct kbase_context *kctx) -+{ -+ const int event_count = kctx->event_coalesce_count; -+ -+ /* Join the list of pending events onto the tail of the main list -+ and reset it */ -+ list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); -+ kctx->event_coalesce_count = 0; -+ -+ /* Return the number of events moved */ -+ return event_count; -+} -+ -+void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) -+{ -+ if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { -+ if (atom->event_code == BASE_JD_EVENT_DONE) { -+ /* Don't report the event */ -+ kbase_event_process_noreport(ctx, atom); -+ return; -+ } -+ } -+ -+ if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { -+ /* Don't report the event */ -+ kbase_event_process_noreport(ctx, atom); -+ return; -+ } -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_POSTED); -+ if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { -+ /* Don't report the event until other event(s) have completed */ -+ mutex_lock(&ctx->event_mutex); -+ list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list); -+ ++ctx->event_coalesce_count; -+ mutex_unlock(&ctx->event_mutex); -+ } else { -+ /* Report the event and any pending events now */ -+ int event_count = 1; -+ -+ mutex_lock(&ctx->event_mutex); -+ event_count += kbase_event_coalesce(ctx); -+ list_add_tail(&atom->dep_item[0], &ctx->event_list); -+ atomic_add(event_count, &ctx->event_count); -+ mutex_unlock(&ctx->event_mutex); -+ -+ kbase_event_wakeup(ctx); -+ } -+} -+KBASE_EXPORT_TEST_API(kbase_event_post); -+ -+void kbase_event_close(struct kbase_context *kctx) -+{ -+ mutex_lock(&kctx->event_mutex); -+ atomic_set(&kctx->event_closed, true); -+ mutex_unlock(&kctx->event_mutex); -+ kbase_event_wakeup(kctx); -+} -+ -+int kbase_event_init(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ INIT_LIST_HEAD(&kctx->event_list); -+ INIT_LIST_HEAD(&kctx->event_coalesce_list); -+ mutex_init(&kctx->event_mutex); -+ atomic_set(&kctx->event_count, 0); -+ kctx->event_coalesce_count = 0; -+ atomic_set(&kctx->event_closed, false); -+ kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); -+ -+ if (NULL == kctx->event_workq) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_event_init); -+ -+void kbase_event_cleanup(struct kbase_context *kctx) -+{ -+ int event_count; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(kctx->event_workq); -+ -+ flush_workqueue(kctx->event_workq); -+ destroy_workqueue(kctx->event_workq); -+ -+ /* We use kbase_event_dequeue to remove the remaining events as that -+ * deals with all the cleanup needed for the atoms. -+ * -+ * Note: use of kctx->event_list without a lock is safe because this must be the last -+ * thread using it (because we're about to terminate the lock) -+ */ -+ event_count = kbase_event_coalesce(kctx); -+ atomic_add(event_count, &kctx->event_count); -+ -+ while (!list_empty(&kctx->event_list)) { -+ struct base_jd_event_v2 event; -+ -+ kbase_event_dequeue(kctx, &event); -+ } -+} -+ -+KBASE_EXPORT_TEST_API(kbase_event_cleanup); -diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence.c b/drivers/gpu/arm/midgard/mali_kbase_fence.c -new file mode 100755 -index 000000000..3bcfb38c3 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_fence.c -@@ -0,0 +1,200 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/* Spin lock protecting all Mali fences as fence->lock. */ -+static DEFINE_SPINLOCK(kbase_fence_lock); -+ -+static const char * -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+kbase_fence_get_driver_name(struct fence *fence) -+#else -+kbase_fence_get_driver_name(struct dma_fence *fence) -+#endif -+{ -+ return kbase_drv_name; -+} -+ -+static const char * -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+kbase_fence_get_timeline_name(struct fence *fence) -+#else -+kbase_fence_get_timeline_name(struct dma_fence *fence) -+#endif -+{ -+ return kbase_timeline_name; -+} -+ -+static bool -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+kbase_fence_enable_signaling(struct fence *fence) -+#else -+kbase_fence_enable_signaling(struct dma_fence *fence) -+#endif -+{ -+ return true; -+} -+ -+static void -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+kbase_fence_fence_value_str(struct fence *fence, char *str, int size) -+#else -+kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) -+#endif -+{ -+#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) -+ snprintf(str, size, "%u", fence->seqno); -+#else -+ snprintf(str, size, "%llu", fence->seqno); -+#endif -+} -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+const struct fence_ops kbase_fence_ops = { -+ .wait = fence_default_wait, -+#else -+const struct dma_fence_ops kbase_fence_ops = { -+ .wait = dma_fence_default_wait, -+#endif -+ .get_driver_name = kbase_fence_get_driver_name, -+ .get_timeline_name = kbase_fence_get_timeline_name, -+ .enable_signaling = kbase_fence_enable_signaling, -+ .fence_value_str = kbase_fence_fence_value_str -+}; -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+struct fence * -+kbase_fence_out_new(struct kbase_jd_atom *katom) -+#else -+struct dma_fence * -+kbase_fence_out_new(struct kbase_jd_atom *katom) -+#endif -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ -+ WARN_ON(katom->dma_fence.fence); -+ -+ fence = kzalloc(sizeof(*fence), GFP_KERNEL); -+ if (!fence) -+ return NULL; -+ -+ dma_fence_init(fence, -+ &kbase_fence_ops, -+ &kbase_fence_lock, -+ katom->dma_fence.context, -+ atomic_inc_return(&katom->dma_fence.seqno)); -+ -+ katom->dma_fence.fence = fence; -+ -+ return fence; -+} -+ -+bool -+kbase_fence_free_callbacks(struct kbase_jd_atom *katom) -+{ -+ struct kbase_fence_cb *cb, *tmp; -+ bool res = false; -+ -+ lockdep_assert_held(&katom->kctx->jctx.lock); -+ -+ /* Clean up and free callbacks. */ -+ list_for_each_entry_safe(cb, tmp, &katom->dma_fence.callbacks, node) { -+ bool ret; -+ -+ /* Cancel callbacks that hasn't been called yet. */ -+ ret = dma_fence_remove_callback(cb->fence, &cb->fence_cb); -+ if (ret) { -+ int ret; -+ -+ /* Fence had not signaled, clean up after -+ * canceling. -+ */ -+ ret = atomic_dec_return(&katom->dma_fence.dep_count); -+ -+ if (unlikely(ret == 0)) -+ res = true; -+ } -+ -+ /* -+ * Release the reference taken in -+ * kbase_fence_add_callback(). -+ */ -+ dma_fence_put(cb->fence); -+ list_del(&cb->node); -+ kfree(cb); -+ } -+ -+ return res; -+} -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+int -+kbase_fence_add_callback(struct kbase_jd_atom *katom, -+ struct fence *fence, -+ fence_func_t callback) -+#else -+int -+kbase_fence_add_callback(struct kbase_jd_atom *katom, -+ struct dma_fence *fence, -+ dma_fence_func_t callback) -+#endif -+{ -+ int err = 0; -+ struct kbase_fence_cb *kbase_fence_cb; -+ -+ if (!fence) -+ return -EINVAL; -+ -+ kbase_fence_cb = kmalloc(sizeof(*kbase_fence_cb), GFP_KERNEL); -+ if (!kbase_fence_cb) -+ return -ENOMEM; -+ -+ kbase_fence_cb->fence = fence; -+ kbase_fence_cb->katom = katom; -+ INIT_LIST_HEAD(&kbase_fence_cb->node); -+ -+ err = dma_fence_add_callback(fence, &kbase_fence_cb->fence_cb, -+ callback); -+ if (err == -ENOENT) { -+ /* Fence signaled, clear the error and return */ -+ err = 0; -+ kfree(kbase_fence_cb); -+ } else if (err) { -+ kfree(kbase_fence_cb); -+ } else { -+ /* -+ * Get reference to fence that will be kept until callback gets -+ * cleaned up in kbase_fence_free_callbacks(). -+ */ -+ dma_fence_get(fence); -+ atomic_inc(&katom->dma_fence.dep_count); -+ /* Add callback to katom's list of callbacks */ -+ list_add(&kbase_fence_cb->node, &katom->dma_fence.callbacks); -+ } -+ -+ return err; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence.h b/drivers/gpu/arm/midgard/mali_kbase_fence.h -new file mode 100755 -index 000000000..639cc2ef4 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_fence.h -@@ -0,0 +1,275 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_FENCE_H_ -+#define _KBASE_FENCE_H_ -+ -+/* -+ * mali_kbase_fence.[hc] has common fence code used by both -+ * - CONFIG_MALI_DMA_FENCE - implicit DMA fences -+ * - CONFIG_SYNC_FILE - explicit fences beginning with 4.9 kernel -+ */ -+ -+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) -+ -+#include -+#include "mali_kbase_fence_defs.h" -+#include "mali_kbase.h" -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+extern const struct fence_ops kbase_fence_ops; -+#else -+extern const struct dma_fence_ops kbase_fence_ops; -+#endif -+ -+/** -+* struct kbase_fence_cb - Mali dma-fence callback data struct -+* @fence_cb: Callback function -+* @katom: Pointer to katom that is waiting on this callback -+* @fence: Pointer to the fence object on which this callback is waiting -+* @node: List head for linking this callback to the katom -+*/ -+struct kbase_fence_cb { -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence_cb fence_cb; -+ struct fence *fence; -+#else -+ struct dma_fence_cb fence_cb; -+ struct dma_fence *fence; -+#endif -+ struct kbase_jd_atom *katom; -+ struct list_head node; -+}; -+ -+/** -+ * kbase_fence_out_new() - Creates a new output fence and puts it on the atom -+ * @katom: Atom to create an output fence for -+ * -+ * return: A new fence object on success, NULL on failure. -+ */ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); -+#else -+struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); -+#endif -+ -+#if defined(CONFIG_SYNC_FILE) -+/** -+ * kbase_fence_fence_in_set() - Assign input fence to atom -+ * @katom: Atom to assign input fence to -+ * @fence: Input fence to assign to atom -+ * -+ * This function will take ownership of one fence reference! -+ */ -+#define kbase_fence_fence_in_set(katom, fence) \ -+ do { \ -+ WARN_ON((katom)->dma_fence.fence_in); \ -+ (katom)->dma_fence.fence_in = fence; \ -+ } while (0) -+#endif -+ -+/** -+ * kbase_fence_out_remove() - Removes the output fence from atom -+ * @katom: Atom to remove output fence for -+ * -+ * This will also release the reference to this fence which the atom keeps -+ */ -+static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) -+{ -+ if (katom->dma_fence.fence) { -+ dma_fence_put(katom->dma_fence.fence); -+ katom->dma_fence.fence = NULL; -+ } -+} -+ -+#if defined(CONFIG_SYNC_FILE) -+/** -+ * kbase_fence_out_remove() - Removes the input fence from atom -+ * @katom: Atom to remove input fence for -+ * -+ * This will also release the reference to this fence which the atom keeps -+ */ -+static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) -+{ -+ if (katom->dma_fence.fence_in) { -+ dma_fence_put(katom->dma_fence.fence_in); -+ katom->dma_fence.fence_in = NULL; -+ } -+} -+#endif -+ -+/** -+ * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us -+ * @katom: Atom to check output fence for -+ * -+ * Return: true if fence exists and is valid, otherwise false -+ */ -+static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) -+{ -+ return katom->dma_fence.fence && -+ katom->dma_fence.fence->ops == &kbase_fence_ops; -+} -+ -+/** -+ * kbase_fence_out_signal() - Signal output fence of atom -+ * @katom: Atom to signal output fence for -+ * @status: Status to signal with (0 for success, < 0 for error) -+ * -+ * Return: 0 on success, < 0 on error -+ */ -+static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, -+ int status) -+{ -+ if (status) { -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ -+ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE) -+ fence_set_error(katom->dma_fence.fence, status); -+#elif (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) -+ dma_fence_set_error(katom->dma_fence.fence, status); -+#else -+ katom->dma_fence.fence->status = status; -+#endif -+ } -+ return dma_fence_signal(katom->dma_fence.fence); -+} -+ -+/** -+ * kbase_fence_add_callback() - Add callback on @fence to block @katom -+ * @katom: Pointer to katom that will be blocked by @fence -+ * @fence: Pointer to fence on which to set up the callback -+ * @callback: Pointer to function to be called when fence is signaled -+ * -+ * Caller needs to hold a reference to @fence when calling this function, and -+ * the caller is responsible for releasing that reference. An additional -+ * reference to @fence will be taken when the callback was successfully set up -+ * and @fence needs to be kept valid until the callback has been called and -+ * cleanup have been done. -+ * -+ * Return: 0 on success: fence was either already signaled, or callback was -+ * set up. Negative error code is returned on error. -+ */ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+int kbase_fence_add_callback(struct kbase_jd_atom *katom, -+ struct fence *fence, -+ fence_func_t callback); -+#else -+int kbase_fence_add_callback(struct kbase_jd_atom *katom, -+ struct dma_fence *fence, -+ dma_fence_func_t callback); -+#endif -+ -+/** -+ * kbase_fence_dep_count_set() - Set dep_count value on atom to specified value -+ * @katom: Atom to set dep_count for -+ * @val: value to set dep_count to -+ * -+ * The dep_count is available to the users of this module so that they can -+ * synchronize completion of the wait with cancellation and adding of more -+ * callbacks. For instance, a user could do the following: -+ * -+ * dep_count set to 1 -+ * callback #1 added, dep_count is increased to 2 -+ * callback #1 happens, dep_count decremented to 1 -+ * since dep_count > 0, no completion is done -+ * callback #2 is added, dep_count is increased to 2 -+ * dep_count decremented to 1 -+ * callback #2 happens, dep_count decremented to 0 -+ * since dep_count now is zero, completion executes -+ * -+ * The dep_count can also be used to make sure that the completion only -+ * executes once. This is typically done by setting dep_count to -1 for the -+ * thread that takes on this responsibility. -+ */ -+static inline void -+kbase_fence_dep_count_set(struct kbase_jd_atom *katom, int val) -+{ -+ atomic_set(&katom->dma_fence.dep_count, val); -+} -+ -+/** -+ * kbase_fence_dep_count_dec_and_test() - Decrements dep_count -+ * @katom: Atom to decrement dep_count for -+ * -+ * See @kbase_fence_dep_count_set for general description about dep_count -+ * -+ * Return: true if value was decremented to zero, otherwise false -+ */ -+static inline bool -+kbase_fence_dep_count_dec_and_test(struct kbase_jd_atom *katom) -+{ -+ return atomic_dec_and_test(&katom->dma_fence.dep_count); -+} -+ -+/** -+ * kbase_fence_dep_count_read() - Returns the current dep_count value -+ * @katom: Pointer to katom -+ * -+ * See @kbase_fence_dep_count_set for general description about dep_count -+ * -+ * Return: The current dep_count value -+ */ -+static inline int kbase_fence_dep_count_read(struct kbase_jd_atom *katom) -+{ -+ return atomic_read(&katom->dma_fence.dep_count); -+} -+ -+/** -+ * kbase_fence_free_callbacks() - Free dma-fence callbacks on a katom -+ * @katom: Pointer to katom -+ * -+ * This function will free all fence callbacks on the katom's list of -+ * callbacks. Callbacks that have not yet been called, because their fence -+ * hasn't yet signaled, will first be removed from the fence. -+ * -+ * Locking: katom->dma_fence.callbacks list assumes jctx.lock is held. -+ * -+ * Return: true if dep_count reached 0, otherwise false. -+ */ -+bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); -+ -+#if defined(CONFIG_SYNC_FILE) -+/** -+ * kbase_fence_in_get() - Retrieve input fence for atom. -+ * @katom: Atom to get input fence from -+ * -+ * A ref will be taken for the fence, so use @kbase_fence_put() to release it -+ * -+ * Return: The fence, or NULL if there is no input fence for atom -+ */ -+#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) -+#endif -+ -+/** -+ * kbase_fence_out_get() - Retrieve output fence for atom. -+ * @katom: Atom to get output fence from -+ * -+ * A ref will be taken for the fence, so use @kbase_fence_put() to release it -+ * -+ * Return: The fence, or NULL if there is no output fence for atom -+ */ -+#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) -+ -+/** -+ * kbase_fence_put() - Releases a reference to a fence -+ * @fence: Fence to release reference for. -+ */ -+#define kbase_fence_put(fence) dma_fence_put(fence) -+ -+ -+#endif /* CONFIG_MALI_DMA_FENCE || defined(CONFIG_SYNC_FILE */ -+ -+#endif /* _KBASE_FENCE_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h b/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h -new file mode 100755 -index 000000000..fa2c6dfe9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h -@@ -0,0 +1,51 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_FENCE_DEFS_H_ -+#define _KBASE_FENCE_DEFS_H_ -+ -+/* -+ * There was a big rename in the 4.10 kernel (fence* -> dma_fence*) -+ * This file hides the compatibility issues with this for the rest the driver -+ */ -+ -+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) -+ -+#include -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ -+#include -+ -+#define dma_fence_context_alloc(a) fence_context_alloc(a) -+#define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) -+#define dma_fence_get(a) fence_get(a) -+#define dma_fence_put(a) fence_put(a) -+#define dma_fence_signal(a) fence_signal(a) -+#define dma_fence_is_signaled(a) fence_is_signaled(a) -+#define dma_fence_add_callback(a, b, c) fence_add_callback(a, b, c) -+#define dma_fence_remove_callback(a, b) fence_remove_callback(a, b) -+ -+#else -+ -+#include -+ -+#endif /* < 4.10.0 */ -+ -+#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */ -+ -+#endif /* _KBASE_FENCE_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator.h b/drivers/gpu/arm/midgard/mali_kbase_gator.h -new file mode 100755 -index 000000000..ce65b5562 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator.h -@@ -0,0 +1,45 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* NB taken from gator */ -+/* -+ * List of possible actions to be controlled by DS-5 Streamline. -+ * The following numbers are used by gator to control the frame buffer dumping -+ * and s/w counter reporting. We cannot use the enums in mali_uk_types.h because -+ * they are unknown inside gator. -+ */ -+#ifndef _KBASE_GATOR_H_ -+#define _KBASE_GATOR_H_ -+ -+#ifdef CONFIG_MALI_GATOR_SUPPORT -+#define GATOR_MAKE_EVENT(type, number) (((type) << 24) | ((number) << 16)) -+#define GATOR_JOB_SLOT_START 1 -+#define GATOR_JOB_SLOT_STOP 2 -+#define GATOR_JOB_SLOT_SOFT_STOPPED 3 -+ -+void kbase_trace_mali_job_slots_event(u32 event, const struct kbase_context *kctx, u8 atom_id); -+void kbase_trace_mali_pm_status(u32 event, u64 value); -+void kbase_trace_mali_pm_power_off(u32 event, u64 value); -+void kbase_trace_mali_pm_power_on(u32 event, u64 value); -+void kbase_trace_mali_page_fault_insert_pages(int event, u32 value); -+void kbase_trace_mali_mmu_as_in_use(int event); -+void kbase_trace_mali_mmu_as_released(int event); -+void kbase_trace_mali_total_alloc_pages_change(long long int event); -+ -+#endif /* CONFIG_MALI_GATOR_SUPPORT */ -+ -+#endif /* _KBASE_GATOR_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_api.c b/drivers/gpu/arm/midgard/mali_kbase_gator_api.c -new file mode 100755 -index 000000000..860e10159 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator_api.c -@@ -0,0 +1,334 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include "mali_kbase.h" -+#include "mali_kbase_hw.h" -+#include "mali_kbase_mem_linux.h" -+#include "mali_kbase_gator_api.h" -+#include "mali_kbase_gator_hwcnt_names.h" -+ -+#define MALI_MAX_CORES_PER_GROUP 4 -+#define MALI_MAX_NUM_BLOCKS_PER_GROUP 8 -+#define MALI_COUNTERS_PER_BLOCK 64 -+#define MALI_BYTES_PER_COUNTER 4 -+ -+struct kbase_gator_hwcnt_handles { -+ struct kbase_device *kbdev; -+ struct kbase_vinstr_client *vinstr_cli; -+ void *vinstr_buffer; -+ struct work_struct dump_work; -+ int dump_complete; -+ spinlock_t dump_lock; -+}; -+ -+static void dump_worker(struct work_struct *work); -+ -+const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters) -+{ -+ const char * const *hardware_counters; -+ struct kbase_device *kbdev; -+ uint32_t product_id; -+ uint32_t count; -+ -+ if (!total_counters) -+ return NULL; -+ -+ /* Get the first device - it doesn't matter in this case */ -+ kbdev = kbase_find_device(-1); -+ if (!kbdev) -+ return NULL; -+ -+ product_id = kbdev->gpu_props.props.core_props.product_id; -+ -+ if (GPU_ID_IS_NEW_FORMAT(product_id)) { -+ switch (GPU_ID2_MODEL_MATCH_VALUE(product_id)) { -+ case GPU_ID2_PRODUCT_TMIX: -+ hardware_counters = hardware_counters_mali_tMIx; -+ count = ARRAY_SIZE(hardware_counters_mali_tMIx); -+ break; -+ case GPU_ID2_PRODUCT_THEX: -+ hardware_counters = hardware_counters_mali_tHEx; -+ count = ARRAY_SIZE(hardware_counters_mali_tHEx); -+ break; -+ case GPU_ID2_PRODUCT_TSIX: -+ hardware_counters = hardware_counters_mali_tSIx; -+ count = ARRAY_SIZE(hardware_counters_mali_tSIx); -+ break; -+ default: -+ hardware_counters = NULL; -+ count = 0; -+ dev_err(kbdev->dev, "Unrecognized product ID: %u\n", -+ product_id); -+ break; -+ } -+ } else { -+ switch (product_id) { -+ /* If we are using a Mali-T60x device */ -+ case GPU_ID_PI_T60X: -+ hardware_counters = hardware_counters_mali_t60x; -+ count = ARRAY_SIZE(hardware_counters_mali_t60x); -+ break; -+ /* If we are using a Mali-T62x device */ -+ case GPU_ID_PI_T62X: -+ hardware_counters = hardware_counters_mali_t62x; -+ count = ARRAY_SIZE(hardware_counters_mali_t62x); -+ break; -+ /* If we are using a Mali-T72x device */ -+ case GPU_ID_PI_T72X: -+ hardware_counters = hardware_counters_mali_t72x; -+ count = ARRAY_SIZE(hardware_counters_mali_t72x); -+ break; -+ /* If we are using a Mali-T76x device */ -+ case GPU_ID_PI_T76X: -+ hardware_counters = hardware_counters_mali_t76x; -+ count = ARRAY_SIZE(hardware_counters_mali_t76x); -+ break; -+ /* If we are using a Mali-T82x device */ -+ case GPU_ID_PI_T82X: -+ hardware_counters = hardware_counters_mali_t82x; -+ count = ARRAY_SIZE(hardware_counters_mali_t82x); -+ break; -+ /* If we are using a Mali-T83x device */ -+ case GPU_ID_PI_T83X: -+ hardware_counters = hardware_counters_mali_t83x; -+ count = ARRAY_SIZE(hardware_counters_mali_t83x); -+ break; -+ /* If we are using a Mali-T86x device */ -+ case GPU_ID_PI_T86X: -+ hardware_counters = hardware_counters_mali_t86x; -+ count = ARRAY_SIZE(hardware_counters_mali_t86x); -+ break; -+ /* If we are using a Mali-T88x device */ -+ case GPU_ID_PI_TFRX: -+ hardware_counters = hardware_counters_mali_t88x; -+ count = ARRAY_SIZE(hardware_counters_mali_t88x); -+ break; -+ default: -+ hardware_counters = NULL; -+ count = 0; -+ dev_err(kbdev->dev, "Unrecognized product ID: %u\n", -+ product_id); -+ break; -+ } -+ } -+ -+ /* Release the kbdev reference. */ -+ kbase_release_device(kbdev); -+ -+ *total_counters = count; -+ -+ /* If we return a string array take a reference on the module (or fail). */ -+ if (hardware_counters && !try_module_get(THIS_MODULE)) -+ return NULL; -+ -+ return hardware_counters; -+} -+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init_names); -+ -+void kbase_gator_hwcnt_term_names(void) -+{ -+ /* Release the module reference. */ -+ module_put(THIS_MODULE); -+} -+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term_names); -+ -+struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info) -+{ -+ struct kbase_gator_hwcnt_handles *hand; -+ struct kbase_uk_hwcnt_reader_setup setup; -+ uint32_t dump_size = 0, i = 0; -+ -+ if (!in_out_info) -+ return NULL; -+ -+ hand = kzalloc(sizeof(*hand), GFP_KERNEL); -+ if (!hand) -+ return NULL; -+ -+ INIT_WORK(&hand->dump_work, dump_worker); -+ spin_lock_init(&hand->dump_lock); -+ -+ /* Get the first device */ -+ hand->kbdev = kbase_find_device(-1); -+ if (!hand->kbdev) -+ goto free_hand; -+ -+ dump_size = kbase_vinstr_dump_size(hand->kbdev); -+ hand->vinstr_buffer = kzalloc(dump_size, GFP_KERNEL); -+ if (!hand->vinstr_buffer) -+ goto release_device; -+ in_out_info->kernel_dump_buffer = hand->vinstr_buffer; -+ -+ in_out_info->nr_cores = hand->kbdev->gpu_props.num_cores; -+ in_out_info->nr_core_groups = hand->kbdev->gpu_props.num_core_groups; -+ in_out_info->gpu_id = hand->kbdev->gpu_props.props.core_props.product_id; -+ -+ /* If we are using a v4 device (Mali-T6xx or Mali-T72x) */ -+ if (kbase_hw_has_feature(hand->kbdev, BASE_HW_FEATURE_V4)) { -+ uint32_t cg, j; -+ uint64_t core_mask; -+ -+ /* There are 8 hardware counters blocks per core group */ -+ in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * -+ MALI_MAX_NUM_BLOCKS_PER_GROUP * -+ in_out_info->nr_core_groups, GFP_KERNEL); -+ -+ if (!in_out_info->hwc_layout) -+ goto free_vinstr_buffer; -+ -+ dump_size = in_out_info->nr_core_groups * -+ MALI_MAX_NUM_BLOCKS_PER_GROUP * -+ MALI_COUNTERS_PER_BLOCK * -+ MALI_BYTES_PER_COUNTER; -+ -+ for (cg = 0; cg < in_out_info->nr_core_groups; cg++) { -+ core_mask = hand->kbdev->gpu_props.props.coherency_info.group[cg].core_mask; -+ -+ for (j = 0; j < MALI_MAX_CORES_PER_GROUP; j++) { -+ if (core_mask & (1u << j)) -+ in_out_info->hwc_layout[i++] = SHADER_BLOCK; -+ else -+ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; -+ } -+ -+ in_out_info->hwc_layout[i++] = TILER_BLOCK; -+ in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; -+ -+ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; -+ -+ if (0 == cg) -+ in_out_info->hwc_layout[i++] = JM_BLOCK; -+ else -+ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; -+ } -+ /* If we are using any other device */ -+ } else { -+ uint32_t nr_l2, nr_sc_bits, j; -+ uint64_t core_mask; -+ -+ nr_l2 = hand->kbdev->gpu_props.props.l2_props.num_l2_slices; -+ -+ core_mask = hand->kbdev->gpu_props.props.coherency_info.group[0].core_mask; -+ -+ nr_sc_bits = fls64(core_mask); -+ -+ /* The job manager and tiler sets of counters -+ * are always present */ -+ in_out_info->hwc_layout = kmalloc(sizeof(enum hwc_type) * (2 + nr_sc_bits + nr_l2), GFP_KERNEL); -+ -+ if (!in_out_info->hwc_layout) -+ goto free_vinstr_buffer; -+ -+ dump_size = (2 + nr_sc_bits + nr_l2) * MALI_COUNTERS_PER_BLOCK * MALI_BYTES_PER_COUNTER; -+ -+ in_out_info->hwc_layout[i++] = JM_BLOCK; -+ in_out_info->hwc_layout[i++] = TILER_BLOCK; -+ -+ for (j = 0; j < nr_l2; j++) -+ in_out_info->hwc_layout[i++] = MMU_L2_BLOCK; -+ -+ while (core_mask != 0ull) { -+ if ((core_mask & 1ull) != 0ull) -+ in_out_info->hwc_layout[i++] = SHADER_BLOCK; -+ else -+ in_out_info->hwc_layout[i++] = RESERVED_BLOCK; -+ core_mask >>= 1; -+ } -+ } -+ -+ in_out_info->nr_hwc_blocks = i; -+ in_out_info->size = dump_size; -+ -+ setup.jm_bm = in_out_info->bitmask[0]; -+ setup.tiler_bm = in_out_info->bitmask[1]; -+ setup.shader_bm = in_out_info->bitmask[2]; -+ setup.mmu_l2_bm = in_out_info->bitmask[3]; -+ hand->vinstr_cli = kbase_vinstr_hwcnt_kernel_setup(hand->kbdev->vinstr_ctx, -+ &setup, hand->vinstr_buffer); -+ if (!hand->vinstr_cli) { -+ dev_err(hand->kbdev->dev, "Failed to register gator with vinstr core"); -+ goto free_layout; -+ } -+ -+ return hand; -+ -+free_layout: -+ kfree(in_out_info->hwc_layout); -+ -+free_vinstr_buffer: -+ kfree(hand->vinstr_buffer); -+ -+release_device: -+ kbase_release_device(hand->kbdev); -+ -+free_hand: -+ kfree(hand); -+ return NULL; -+} -+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_init); -+ -+void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles) -+{ -+ if (in_out_info) -+ kfree(in_out_info->hwc_layout); -+ -+ if (opaque_handles) { -+ cancel_work_sync(&opaque_handles->dump_work); -+ kbase_vinstr_detach_client(opaque_handles->vinstr_cli); -+ kfree(opaque_handles->vinstr_buffer); -+ kbase_release_device(opaque_handles->kbdev); -+ kfree(opaque_handles); -+ } -+} -+KBASE_EXPORT_SYMBOL(kbase_gator_hwcnt_term); -+ -+static void dump_worker(struct work_struct *work) -+{ -+ struct kbase_gator_hwcnt_handles *hand; -+ -+ hand = container_of(work, struct kbase_gator_hwcnt_handles, dump_work); -+ if (!kbase_vinstr_hwc_dump(hand->vinstr_cli, -+ BASE_HWCNT_READER_EVENT_MANUAL)) { -+ spin_lock_bh(&hand->dump_lock); -+ hand->dump_complete = 1; -+ spin_unlock_bh(&hand->dump_lock); -+ } else { -+ schedule_work(&hand->dump_work); -+ } -+} -+ -+uint32_t kbase_gator_instr_hwcnt_dump_complete( -+ struct kbase_gator_hwcnt_handles *opaque_handles, -+ uint32_t * const success) -+{ -+ -+ if (opaque_handles && success) { -+ *success = opaque_handles->dump_complete; -+ opaque_handles->dump_complete = 0; -+ return *success; -+ } -+ return 0; -+} -+KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_complete); -+ -+uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles) -+{ -+ if (opaque_handles) -+ schedule_work(&opaque_handles->dump_work); -+ return 0; -+} -+KBASE_EXPORT_SYMBOL(kbase_gator_instr_hwcnt_dump_irq); -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_api.h b/drivers/gpu/arm/midgard/mali_kbase_gator_api.h -new file mode 100755 -index 000000000..ef9ac0f7b ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator_api.h -@@ -0,0 +1,219 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_GATOR_API_H_ -+#define _KBASE_GATOR_API_H_ -+ -+/** -+ * @brief This file describes the API used by Gator to fetch hardware counters. -+ */ -+ -+/* This define is used by the gator kernel module compile to select which DDK -+ * API calling convention to use. If not defined (legacy DDK) gator assumes -+ * version 1. The version to DDK release mapping is: -+ * Version 1 API: DDK versions r1px, r2px -+ * Version 2 API: DDK versions r3px, r4px -+ * Version 3 API: DDK version r5p0 and newer -+ * -+ * API Usage -+ * ========= -+ * -+ * 1] Call kbase_gator_hwcnt_init_names() to return the list of short counter -+ * names for the GPU present in this device. -+ * -+ * 2] Create a kbase_gator_hwcnt_info structure and set the counter enables for -+ * the counters you want enabled. The enables can all be set for simplicity in -+ * most use cases, but disabling some will let you minimize bandwidth impact. -+ * -+ * 3] Call kbase_gator_hwcnt_init() using the above structure, to create a -+ * counter context. On successful return the DDK will have populated the -+ * structure with a variety of useful information. -+ * -+ * 4] Call kbase_gator_hwcnt_dump_irq() to queue a non-blocking request for a -+ * counter dump. If this returns a non-zero value the request has been queued, -+ * otherwise the driver has been unable to do so (typically because of another -+ * user of the instrumentation exists concurrently). -+ * -+ * 5] Call kbase_gator_hwcnt_dump_complete() to test whether the previously -+ * requested dump has been succesful. If this returns non-zero the counter dump -+ * has resolved, but the value of *success must also be tested as the dump -+ * may have not been successful. If it returns zero the counter dump was -+ * abandoned due to the device being busy (typically because of another -+ * user of the instrumentation exists concurrently). -+ * -+ * 6] Process the counters stored in the buffer pointed to by ... -+ * -+ * kbase_gator_hwcnt_info->kernel_dump_buffer -+ * -+ * In pseudo code you can find all of the counters via this approach: -+ * -+ * -+ * hwcnt_info # pointer to kbase_gator_hwcnt_info structure -+ * hwcnt_name # pointer to name list -+ * -+ * u32 * hwcnt_data = (u32*)hwcnt_info->kernel_dump_buffer -+ * -+ * # Iterate over each 64-counter block in this GPU configuration -+ * for( i = 0; i < hwcnt_info->nr_hwc_blocks; i++) { -+ * hwc_type type = hwcnt_info->hwc_layout[i]; -+ * -+ * # Skip reserved type blocks - they contain no counters at all -+ * if( type == RESERVED_BLOCK ) { -+ * continue; -+ * } -+ * -+ * size_t name_offset = type * 64; -+ * size_t data_offset = i * 64; -+ * -+ * # Iterate over the names of the counters in this block type -+ * for( j = 0; j < 64; j++) { -+ * const char * name = hwcnt_name[name_offset+j]; -+ * -+ * # Skip empty name strings - there is no counter here -+ * if( name[0] == '\0' ) { -+ * continue; -+ * } -+ * -+ * u32 data = hwcnt_data[data_offset+j]; -+ * -+ * printk( "COUNTER: %s DATA: %u\n", name, data ); -+ * } -+ * } -+ * -+ * -+ * Note that in most implementations you typically want to either SUM or -+ * AVERAGE multiple instances of the same counter if, for example, you have -+ * multiple shader cores or multiple L2 caches. The most sensible view for -+ * analysis is to AVERAGE shader core counters, but SUM L2 cache and MMU -+ * counters. -+ * -+ * 7] Goto 4, repeating until you want to stop collecting counters. -+ * -+ * 8] Release the dump resources by calling kbase_gator_hwcnt_term(). -+ * -+ * 9] Release the name table resources by calling -+ * kbase_gator_hwcnt_term_names(). This function must only be called if -+ * init_names() returned a non-NULL value. -+ **/ -+ -+#define MALI_DDK_GATOR_API_VERSION 3 -+ -+enum hwc_type { -+ JM_BLOCK = 0, -+ TILER_BLOCK, -+ SHADER_BLOCK, -+ MMU_L2_BLOCK, -+ RESERVED_BLOCK -+}; -+ -+struct kbase_gator_hwcnt_info { -+ /* Passed from Gator to kbase */ -+ -+ /* the bitmask of enabled hardware counters for each counter block */ -+ uint16_t bitmask[4]; -+ -+ /* Passed from kbase to Gator */ -+ -+ /* ptr to counter dump memory */ -+ void *kernel_dump_buffer; -+ -+ /* size of counter dump memory */ -+ uint32_t size; -+ -+ /* the ID of the Mali device */ -+ uint32_t gpu_id; -+ -+ /* the number of shader cores in the GPU */ -+ uint32_t nr_cores; -+ -+ /* the number of core groups */ -+ uint32_t nr_core_groups; -+ -+ /* the memory layout of the performance counters */ -+ enum hwc_type *hwc_layout; -+ -+ /* the total number of hardware couter blocks */ -+ uint32_t nr_hwc_blocks; -+}; -+ -+/** -+ * @brief Opaque block of Mali data which Gator needs to return to the API later. -+ */ -+struct kbase_gator_hwcnt_handles; -+ -+/** -+ * @brief Initialize the resources Gator needs for performance profiling. -+ * -+ * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the Mali -+ * specific information that will be returned to Gator. On entry Gator must have populated the -+ * 'bitmask' field with the counters it wishes to enable for each class of counter block. -+ * Each entry in the array corresponds to a single counter class based on the "hwc_type" -+ * enumeration, and each bit corresponds to an enable for 4 sequential counters (LSB enables -+ * the first 4 counters in the block, and so on). See the GPU counter array as returned by -+ * kbase_gator_hwcnt_get_names() for the index values of each counter for the curernt GPU. -+ * -+ * @return Pointer to an opaque handle block on success, NULL on error. -+ */ -+extern struct kbase_gator_hwcnt_handles *kbase_gator_hwcnt_init(struct kbase_gator_hwcnt_info *in_out_info); -+ -+/** -+ * @brief Free all resources once Gator has finished using performance counters. -+ * -+ * @param in_out_info A pointer to a structure containing the enabled counters passed from Gator and all the -+ * Mali specific information that will be returned to Gator. -+ * @param opaque_handles A wrapper structure for kbase structures. -+ */ -+extern void kbase_gator_hwcnt_term(struct kbase_gator_hwcnt_info *in_out_info, struct kbase_gator_hwcnt_handles *opaque_handles); -+ -+/** -+ * @brief Poll whether a counter dump is successful. -+ * -+ * @param opaque_handles A wrapper structure for kbase structures. -+ * @param[out] success Non-zero on success, zero on failure. -+ * -+ * @return Zero if the dump is still pending, non-zero if the dump has completed. Note that a -+ * completed dump may not have dumped succesfully, so the caller must test for both -+ * a completed and successful dump before processing counters. -+ */ -+extern uint32_t kbase_gator_instr_hwcnt_dump_complete(struct kbase_gator_hwcnt_handles *opaque_handles, uint32_t * const success); -+ -+/** -+ * @brief Request the generation of a new counter dump. -+ * -+ * @param opaque_handles A wrapper structure for kbase structures. -+ * -+ * @return Zero if the hardware device is busy and cannot handle the request, non-zero otherwise. -+ */ -+extern uint32_t kbase_gator_instr_hwcnt_dump_irq(struct kbase_gator_hwcnt_handles *opaque_handles); -+ -+/** -+ * @brief This function is used to fetch the names table based on the Mali device in use. -+ * -+ * @param[out] total_counters The total number of counters short names in the Mali devices' list. -+ * -+ * @return Pointer to an array of strings of length *total_counters. -+ */ -+extern const char * const *kbase_gator_hwcnt_init_names(uint32_t *total_counters); -+ -+/** -+ * @brief This function is used to terminate the use of the names table. -+ * -+ * This function must only be called if the initial call to kbase_gator_hwcnt_init_names returned a non-NULL value. -+ */ -+extern void kbase_gator_hwcnt_term_names(void); -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h -new file mode 100755 -index 000000000..cad19b662 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names.h -@@ -0,0 +1,2170 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_GATOR_HWCNT_NAMES_H_ -+#define _KBASE_GATOR_HWCNT_NAMES_H_ -+ -+/* -+ * "Short names" for hardware counters used by Streamline. Counters names are -+ * stored in accordance with their memory layout in the binary counter block -+ * emitted by the Mali GPU. Each "master" in the GPU emits a fixed-size block -+ * of 64 counters, and each GPU implements the same set of "masters" although -+ * the counters each master exposes within its block of 64 may vary. -+ * -+ * Counters which are an empty string are simply "holes" in the counter memory -+ * where no counter exists. -+ */ -+ -+static const char * const hardware_counters_mali_t60x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T60x_MESSAGES_SENT", -+ "T60x_MESSAGES_RECEIVED", -+ "T60x_GPU_ACTIVE", -+ "T60x_IRQ_ACTIVE", -+ "T60x_JS0_JOBS", -+ "T60x_JS0_TASKS", -+ "T60x_JS0_ACTIVE", -+ "", -+ "T60x_JS0_WAIT_READ", -+ "T60x_JS0_WAIT_ISSUE", -+ "T60x_JS0_WAIT_DEPEND", -+ "T60x_JS0_WAIT_FINISH", -+ "T60x_JS1_JOBS", -+ "T60x_JS1_TASKS", -+ "T60x_JS1_ACTIVE", -+ "", -+ "T60x_JS1_WAIT_READ", -+ "T60x_JS1_WAIT_ISSUE", -+ "T60x_JS1_WAIT_DEPEND", -+ "T60x_JS1_WAIT_FINISH", -+ "T60x_JS2_JOBS", -+ "T60x_JS2_TASKS", -+ "T60x_JS2_ACTIVE", -+ "", -+ "T60x_JS2_WAIT_READ", -+ "T60x_JS2_WAIT_ISSUE", -+ "T60x_JS2_WAIT_DEPEND", -+ "T60x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T60x_TI_JOBS_PROCESSED", -+ "T60x_TI_TRIANGLES", -+ "T60x_TI_QUADS", -+ "T60x_TI_POLYGONS", -+ "T60x_TI_POINTS", -+ "T60x_TI_LINES", -+ "T60x_TI_VCACHE_HIT", -+ "T60x_TI_VCACHE_MISS", -+ "T60x_TI_FRONT_FACING", -+ "T60x_TI_BACK_FACING", -+ "T60x_TI_PRIM_VISIBLE", -+ "T60x_TI_PRIM_CULLED", -+ "T60x_TI_PRIM_CLIPPED", -+ "T60x_TI_LEVEL0", -+ "T60x_TI_LEVEL1", -+ "T60x_TI_LEVEL2", -+ "T60x_TI_LEVEL3", -+ "T60x_TI_LEVEL4", -+ "T60x_TI_LEVEL5", -+ "T60x_TI_LEVEL6", -+ "T60x_TI_LEVEL7", -+ "T60x_TI_COMMAND_1", -+ "T60x_TI_COMMAND_2", -+ "T60x_TI_COMMAND_3", -+ "T60x_TI_COMMAND_4", -+ "T60x_TI_COMMAND_4_7", -+ "T60x_TI_COMMAND_8_15", -+ "T60x_TI_COMMAND_16_63", -+ "T60x_TI_COMMAND_64", -+ "T60x_TI_COMPRESS_IN", -+ "T60x_TI_COMPRESS_OUT", -+ "T60x_TI_COMPRESS_FLUSH", -+ "T60x_TI_TIMESTAMPS", -+ "T60x_TI_PCACHE_HIT", -+ "T60x_TI_PCACHE_MISS", -+ "T60x_TI_PCACHE_LINE", -+ "T60x_TI_PCACHE_STALL", -+ "T60x_TI_WRBUF_HIT", -+ "T60x_TI_WRBUF_MISS", -+ "T60x_TI_WRBUF_LINE", -+ "T60x_TI_WRBUF_PARTIAL", -+ "T60x_TI_WRBUF_STALL", -+ "T60x_TI_ACTIVE", -+ "T60x_TI_LOADING_DESC", -+ "T60x_TI_INDEX_WAIT", -+ "T60x_TI_INDEX_RANGE_WAIT", -+ "T60x_TI_VERTEX_WAIT", -+ "T60x_TI_PCACHE_WAIT", -+ "T60x_TI_WRBUF_WAIT", -+ "T60x_TI_BUS_READ", -+ "T60x_TI_BUS_WRITE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T60x_TI_UTLB_STALL", -+ "T60x_TI_UTLB_REPLAY_MISS", -+ "T60x_TI_UTLB_REPLAY_FULL", -+ "T60x_TI_UTLB_NEW_MISS", -+ "T60x_TI_UTLB_HIT", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T60x_FRAG_ACTIVE", -+ "T60x_FRAG_PRIMITIVES", -+ "T60x_FRAG_PRIMITIVES_DROPPED", -+ "T60x_FRAG_CYCLES_DESC", -+ "T60x_FRAG_CYCLES_PLR", -+ "T60x_FRAG_CYCLES_VERT", -+ "T60x_FRAG_CYCLES_TRISETUP", -+ "T60x_FRAG_CYCLES_RAST", -+ "T60x_FRAG_THREADS", -+ "T60x_FRAG_DUMMY_THREADS", -+ "T60x_FRAG_QUADS_RAST", -+ "T60x_FRAG_QUADS_EZS_TEST", -+ "T60x_FRAG_QUADS_EZS_KILLED", -+ "T60x_FRAG_THREADS_LZS_TEST", -+ "T60x_FRAG_THREADS_LZS_KILLED", -+ "T60x_FRAG_CYCLES_NO_TILE", -+ "T60x_FRAG_NUM_TILES", -+ "T60x_FRAG_TRANS_ELIM", -+ "T60x_COMPUTE_ACTIVE", -+ "T60x_COMPUTE_TASKS", -+ "T60x_COMPUTE_THREADS", -+ "T60x_COMPUTE_CYCLES_DESC", -+ "T60x_TRIPIPE_ACTIVE", -+ "T60x_ARITH_WORDS", -+ "T60x_ARITH_CYCLES_REG", -+ "T60x_ARITH_CYCLES_L0", -+ "T60x_ARITH_FRAG_DEPEND", -+ "T60x_LS_WORDS", -+ "T60x_LS_ISSUES", -+ "T60x_LS_RESTARTS", -+ "T60x_LS_REISSUES_MISS", -+ "T60x_LS_REISSUES_VD", -+ "T60x_LS_REISSUE_ATTRIB_MISS", -+ "T60x_LS_NO_WB", -+ "T60x_TEX_WORDS", -+ "T60x_TEX_BUBBLES", -+ "T60x_TEX_WORDS_L0", -+ "T60x_TEX_WORDS_DESC", -+ "T60x_TEX_ISSUES", -+ "T60x_TEX_RECIRC_FMISS", -+ "T60x_TEX_RECIRC_DESC", -+ "T60x_TEX_RECIRC_MULTI", -+ "T60x_TEX_RECIRC_PMISS", -+ "T60x_TEX_RECIRC_CONF", -+ "T60x_LSC_READ_HITS", -+ "T60x_LSC_READ_MISSES", -+ "T60x_LSC_WRITE_HITS", -+ "T60x_LSC_WRITE_MISSES", -+ "T60x_LSC_ATOMIC_HITS", -+ "T60x_LSC_ATOMIC_MISSES", -+ "T60x_LSC_LINE_FETCHES", -+ "T60x_LSC_DIRTY_LINE", -+ "T60x_LSC_SNOOPS", -+ "T60x_AXI_TLB_STALL", -+ "T60x_AXI_TLB_MISS", -+ "T60x_AXI_TLB_TRANSACTION", -+ "T60x_LS_TLB_MISS", -+ "T60x_LS_TLB_HIT", -+ "T60x_AXI_BEATS_READ", -+ "T60x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T60x_MMU_HIT", -+ "T60x_MMU_NEW_MISS", -+ "T60x_MMU_REPLAY_FULL", -+ "T60x_MMU_REPLAY_MISS", -+ "T60x_MMU_TABLE_WALK", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T60x_UTLB_HIT", -+ "T60x_UTLB_NEW_MISS", -+ "T60x_UTLB_REPLAY_FULL", -+ "T60x_UTLB_REPLAY_MISS", -+ "T60x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T60x_L2_EXT_WRITE_BEATS", -+ "T60x_L2_EXT_READ_BEATS", -+ "T60x_L2_ANY_LOOKUP", -+ "T60x_L2_READ_LOOKUP", -+ "T60x_L2_SREAD_LOOKUP", -+ "T60x_L2_READ_REPLAY", -+ "T60x_L2_READ_SNOOP", -+ "T60x_L2_READ_HIT", -+ "T60x_L2_CLEAN_MISS", -+ "T60x_L2_WRITE_LOOKUP", -+ "T60x_L2_SWRITE_LOOKUP", -+ "T60x_L2_WRITE_REPLAY", -+ "T60x_L2_WRITE_SNOOP", -+ "T60x_L2_WRITE_HIT", -+ "T60x_L2_EXT_READ_FULL", -+ "T60x_L2_EXT_READ_HALF", -+ "T60x_L2_EXT_WRITE_FULL", -+ "T60x_L2_EXT_WRITE_HALF", -+ "T60x_L2_EXT_READ", -+ "T60x_L2_EXT_READ_LINE", -+ "T60x_L2_EXT_WRITE", -+ "T60x_L2_EXT_WRITE_LINE", -+ "T60x_L2_EXT_WRITE_SMALL", -+ "T60x_L2_EXT_BARRIER", -+ "T60x_L2_EXT_AR_STALL", -+ "T60x_L2_EXT_R_BUF_FULL", -+ "T60x_L2_EXT_RD_BUF_FULL", -+ "T60x_L2_EXT_R_RAW", -+ "T60x_L2_EXT_W_STALL", -+ "T60x_L2_EXT_W_BUF_FULL", -+ "T60x_L2_EXT_R_W_HAZARD", -+ "T60x_L2_TAG_HAZARD", -+ "T60x_L2_SNOOP_FULL", -+ "T60x_L2_REPLAY_FULL" -+}; -+static const char * const hardware_counters_mali_t62x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T62x_MESSAGES_SENT", -+ "T62x_MESSAGES_RECEIVED", -+ "T62x_GPU_ACTIVE", -+ "T62x_IRQ_ACTIVE", -+ "T62x_JS0_JOBS", -+ "T62x_JS0_TASKS", -+ "T62x_JS0_ACTIVE", -+ "", -+ "T62x_JS0_WAIT_READ", -+ "T62x_JS0_WAIT_ISSUE", -+ "T62x_JS0_WAIT_DEPEND", -+ "T62x_JS0_WAIT_FINISH", -+ "T62x_JS1_JOBS", -+ "T62x_JS1_TASKS", -+ "T62x_JS1_ACTIVE", -+ "", -+ "T62x_JS1_WAIT_READ", -+ "T62x_JS1_WAIT_ISSUE", -+ "T62x_JS1_WAIT_DEPEND", -+ "T62x_JS1_WAIT_FINISH", -+ "T62x_JS2_JOBS", -+ "T62x_JS2_TASKS", -+ "T62x_JS2_ACTIVE", -+ "", -+ "T62x_JS2_WAIT_READ", -+ "T62x_JS2_WAIT_ISSUE", -+ "T62x_JS2_WAIT_DEPEND", -+ "T62x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T62x_TI_JOBS_PROCESSED", -+ "T62x_TI_TRIANGLES", -+ "T62x_TI_QUADS", -+ "T62x_TI_POLYGONS", -+ "T62x_TI_POINTS", -+ "T62x_TI_LINES", -+ "T62x_TI_VCACHE_HIT", -+ "T62x_TI_VCACHE_MISS", -+ "T62x_TI_FRONT_FACING", -+ "T62x_TI_BACK_FACING", -+ "T62x_TI_PRIM_VISIBLE", -+ "T62x_TI_PRIM_CULLED", -+ "T62x_TI_PRIM_CLIPPED", -+ "T62x_TI_LEVEL0", -+ "T62x_TI_LEVEL1", -+ "T62x_TI_LEVEL2", -+ "T62x_TI_LEVEL3", -+ "T62x_TI_LEVEL4", -+ "T62x_TI_LEVEL5", -+ "T62x_TI_LEVEL6", -+ "T62x_TI_LEVEL7", -+ "T62x_TI_COMMAND_1", -+ "T62x_TI_COMMAND_2", -+ "T62x_TI_COMMAND_3", -+ "T62x_TI_COMMAND_4", -+ "T62x_TI_COMMAND_5_7", -+ "T62x_TI_COMMAND_8_15", -+ "T62x_TI_COMMAND_16_63", -+ "T62x_TI_COMMAND_64", -+ "T62x_TI_COMPRESS_IN", -+ "T62x_TI_COMPRESS_OUT", -+ "T62x_TI_COMPRESS_FLUSH", -+ "T62x_TI_TIMESTAMPS", -+ "T62x_TI_PCACHE_HIT", -+ "T62x_TI_PCACHE_MISS", -+ "T62x_TI_PCACHE_LINE", -+ "T62x_TI_PCACHE_STALL", -+ "T62x_TI_WRBUF_HIT", -+ "T62x_TI_WRBUF_MISS", -+ "T62x_TI_WRBUF_LINE", -+ "T62x_TI_WRBUF_PARTIAL", -+ "T62x_TI_WRBUF_STALL", -+ "T62x_TI_ACTIVE", -+ "T62x_TI_LOADING_DESC", -+ "T62x_TI_INDEX_WAIT", -+ "T62x_TI_INDEX_RANGE_WAIT", -+ "T62x_TI_VERTEX_WAIT", -+ "T62x_TI_PCACHE_WAIT", -+ "T62x_TI_WRBUF_WAIT", -+ "T62x_TI_BUS_READ", -+ "T62x_TI_BUS_WRITE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T62x_TI_UTLB_STALL", -+ "T62x_TI_UTLB_REPLAY_MISS", -+ "T62x_TI_UTLB_REPLAY_FULL", -+ "T62x_TI_UTLB_NEW_MISS", -+ "T62x_TI_UTLB_HIT", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "T62x_SHADER_CORE_ACTIVE", -+ "T62x_FRAG_ACTIVE", -+ "T62x_FRAG_PRIMITIVES", -+ "T62x_FRAG_PRIMITIVES_DROPPED", -+ "T62x_FRAG_CYCLES_DESC", -+ "T62x_FRAG_CYCLES_FPKQ_ACTIVE", -+ "T62x_FRAG_CYCLES_VERT", -+ "T62x_FRAG_CYCLES_TRISETUP", -+ "T62x_FRAG_CYCLES_EZS_ACTIVE", -+ "T62x_FRAG_THREADS", -+ "T62x_FRAG_DUMMY_THREADS", -+ "T62x_FRAG_QUADS_RAST", -+ "T62x_FRAG_QUADS_EZS_TEST", -+ "T62x_FRAG_QUADS_EZS_KILLED", -+ "T62x_FRAG_THREADS_LZS_TEST", -+ "T62x_FRAG_THREADS_LZS_KILLED", -+ "T62x_FRAG_CYCLES_NO_TILE", -+ "T62x_FRAG_NUM_TILES", -+ "T62x_FRAG_TRANS_ELIM", -+ "T62x_COMPUTE_ACTIVE", -+ "T62x_COMPUTE_TASKS", -+ "T62x_COMPUTE_THREADS", -+ "T62x_COMPUTE_CYCLES_DESC", -+ "T62x_TRIPIPE_ACTIVE", -+ "T62x_ARITH_WORDS", -+ "T62x_ARITH_CYCLES_REG", -+ "T62x_ARITH_CYCLES_L0", -+ "T62x_ARITH_FRAG_DEPEND", -+ "T62x_LS_WORDS", -+ "T62x_LS_ISSUES", -+ "T62x_LS_RESTARTS", -+ "T62x_LS_REISSUES_MISS", -+ "T62x_LS_REISSUES_VD", -+ "T62x_LS_REISSUE_ATTRIB_MISS", -+ "T62x_LS_NO_WB", -+ "T62x_TEX_WORDS", -+ "T62x_TEX_BUBBLES", -+ "T62x_TEX_WORDS_L0", -+ "T62x_TEX_WORDS_DESC", -+ "T62x_TEX_ISSUES", -+ "T62x_TEX_RECIRC_FMISS", -+ "T62x_TEX_RECIRC_DESC", -+ "T62x_TEX_RECIRC_MULTI", -+ "T62x_TEX_RECIRC_PMISS", -+ "T62x_TEX_RECIRC_CONF", -+ "T62x_LSC_READ_HITS", -+ "T62x_LSC_READ_MISSES", -+ "T62x_LSC_WRITE_HITS", -+ "T62x_LSC_WRITE_MISSES", -+ "T62x_LSC_ATOMIC_HITS", -+ "T62x_LSC_ATOMIC_MISSES", -+ "T62x_LSC_LINE_FETCHES", -+ "T62x_LSC_DIRTY_LINE", -+ "T62x_LSC_SNOOPS", -+ "T62x_AXI_TLB_STALL", -+ "T62x_AXI_TLB_MISS", -+ "T62x_AXI_TLB_TRANSACTION", -+ "T62x_LS_TLB_MISS", -+ "T62x_LS_TLB_HIT", -+ "T62x_AXI_BEATS_READ", -+ "T62x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T62x_MMU_HIT", -+ "T62x_MMU_NEW_MISS", -+ "T62x_MMU_REPLAY_FULL", -+ "T62x_MMU_REPLAY_MISS", -+ "T62x_MMU_TABLE_WALK", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T62x_UTLB_HIT", -+ "T62x_UTLB_NEW_MISS", -+ "T62x_UTLB_REPLAY_FULL", -+ "T62x_UTLB_REPLAY_MISS", -+ "T62x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T62x_L2_EXT_WRITE_BEATS", -+ "T62x_L2_EXT_READ_BEATS", -+ "T62x_L2_ANY_LOOKUP", -+ "T62x_L2_READ_LOOKUP", -+ "T62x_L2_SREAD_LOOKUP", -+ "T62x_L2_READ_REPLAY", -+ "T62x_L2_READ_SNOOP", -+ "T62x_L2_READ_HIT", -+ "T62x_L2_CLEAN_MISS", -+ "T62x_L2_WRITE_LOOKUP", -+ "T62x_L2_SWRITE_LOOKUP", -+ "T62x_L2_WRITE_REPLAY", -+ "T62x_L2_WRITE_SNOOP", -+ "T62x_L2_WRITE_HIT", -+ "T62x_L2_EXT_READ_FULL", -+ "T62x_L2_EXT_READ_HALF", -+ "T62x_L2_EXT_WRITE_FULL", -+ "T62x_L2_EXT_WRITE_HALF", -+ "T62x_L2_EXT_READ", -+ "T62x_L2_EXT_READ_LINE", -+ "T62x_L2_EXT_WRITE", -+ "T62x_L2_EXT_WRITE_LINE", -+ "T62x_L2_EXT_WRITE_SMALL", -+ "T62x_L2_EXT_BARRIER", -+ "T62x_L2_EXT_AR_STALL", -+ "T62x_L2_EXT_R_BUF_FULL", -+ "T62x_L2_EXT_RD_BUF_FULL", -+ "T62x_L2_EXT_R_RAW", -+ "T62x_L2_EXT_W_STALL", -+ "T62x_L2_EXT_W_BUF_FULL", -+ "T62x_L2_EXT_R_W_HAZARD", -+ "T62x_L2_TAG_HAZARD", -+ "T62x_L2_SNOOP_FULL", -+ "T62x_L2_REPLAY_FULL" -+}; -+ -+static const char * const hardware_counters_mali_t72x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T72x_GPU_ACTIVE", -+ "T72x_IRQ_ACTIVE", -+ "T72x_JS0_JOBS", -+ "T72x_JS0_TASKS", -+ "T72x_JS0_ACTIVE", -+ "T72x_JS1_JOBS", -+ "T72x_JS1_TASKS", -+ "T72x_JS1_ACTIVE", -+ "T72x_JS2_JOBS", -+ "T72x_JS2_TASKS", -+ "T72x_JS2_ACTIVE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T72x_TI_JOBS_PROCESSED", -+ "T72x_TI_TRIANGLES", -+ "T72x_TI_QUADS", -+ "T72x_TI_POLYGONS", -+ "T72x_TI_POINTS", -+ "T72x_TI_LINES", -+ "T72x_TI_FRONT_FACING", -+ "T72x_TI_BACK_FACING", -+ "T72x_TI_PRIM_VISIBLE", -+ "T72x_TI_PRIM_CULLED", -+ "T72x_TI_PRIM_CLIPPED", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T72x_TI_ACTIVE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T72x_FRAG_ACTIVE", -+ "T72x_FRAG_PRIMITIVES", -+ "T72x_FRAG_PRIMITIVES_DROPPED", -+ "T72x_FRAG_THREADS", -+ "T72x_FRAG_DUMMY_THREADS", -+ "T72x_FRAG_QUADS_RAST", -+ "T72x_FRAG_QUADS_EZS_TEST", -+ "T72x_FRAG_QUADS_EZS_KILLED", -+ "T72x_FRAG_THREADS_LZS_TEST", -+ "T72x_FRAG_THREADS_LZS_KILLED", -+ "T72x_FRAG_CYCLES_NO_TILE", -+ "T72x_FRAG_NUM_TILES", -+ "T72x_FRAG_TRANS_ELIM", -+ "T72x_COMPUTE_ACTIVE", -+ "T72x_COMPUTE_TASKS", -+ "T72x_COMPUTE_THREADS", -+ "T72x_TRIPIPE_ACTIVE", -+ "T72x_ARITH_WORDS", -+ "T72x_ARITH_CYCLES_REG", -+ "T72x_LS_WORDS", -+ "T72x_LS_ISSUES", -+ "T72x_LS_RESTARTS", -+ "T72x_LS_REISSUES_MISS", -+ "T72x_TEX_WORDS", -+ "T72x_TEX_BUBBLES", -+ "T72x_TEX_ISSUES", -+ "T72x_LSC_READ_HITS", -+ "T72x_LSC_READ_MISSES", -+ "T72x_LSC_WRITE_HITS", -+ "T72x_LSC_WRITE_MISSES", -+ "T72x_LSC_ATOMIC_HITS", -+ "T72x_LSC_ATOMIC_MISSES", -+ "T72x_LSC_LINE_FETCHES", -+ "T72x_LSC_DIRTY_LINE", -+ "T72x_LSC_SNOOPS", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T72x_L2_EXT_WRITE_BEAT", -+ "T72x_L2_EXT_READ_BEAT", -+ "T72x_L2_READ_SNOOP", -+ "T72x_L2_READ_HIT", -+ "T72x_L2_WRITE_SNOOP", -+ "T72x_L2_WRITE_HIT", -+ "T72x_L2_EXT_WRITE_SMALL", -+ "T72x_L2_EXT_BARRIER", -+ "T72x_L2_EXT_AR_STALL", -+ "T72x_L2_EXT_W_STALL", -+ "T72x_L2_SNOOP_FULL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "" -+}; -+ -+static const char * const hardware_counters_mali_t76x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T76x_MESSAGES_SENT", -+ "T76x_MESSAGES_RECEIVED", -+ "T76x_GPU_ACTIVE", -+ "T76x_IRQ_ACTIVE", -+ "T76x_JS0_JOBS", -+ "T76x_JS0_TASKS", -+ "T76x_JS0_ACTIVE", -+ "", -+ "T76x_JS0_WAIT_READ", -+ "T76x_JS0_WAIT_ISSUE", -+ "T76x_JS0_WAIT_DEPEND", -+ "T76x_JS0_WAIT_FINISH", -+ "T76x_JS1_JOBS", -+ "T76x_JS1_TASKS", -+ "T76x_JS1_ACTIVE", -+ "", -+ "T76x_JS1_WAIT_READ", -+ "T76x_JS1_WAIT_ISSUE", -+ "T76x_JS1_WAIT_DEPEND", -+ "T76x_JS1_WAIT_FINISH", -+ "T76x_JS2_JOBS", -+ "T76x_JS2_TASKS", -+ "T76x_JS2_ACTIVE", -+ "", -+ "T76x_JS2_WAIT_READ", -+ "T76x_JS2_WAIT_ISSUE", -+ "T76x_JS2_WAIT_DEPEND", -+ "T76x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T76x_TI_JOBS_PROCESSED", -+ "T76x_TI_TRIANGLES", -+ "T76x_TI_QUADS", -+ "T76x_TI_POLYGONS", -+ "T76x_TI_POINTS", -+ "T76x_TI_LINES", -+ "T76x_TI_VCACHE_HIT", -+ "T76x_TI_VCACHE_MISS", -+ "T76x_TI_FRONT_FACING", -+ "T76x_TI_BACK_FACING", -+ "T76x_TI_PRIM_VISIBLE", -+ "T76x_TI_PRIM_CULLED", -+ "T76x_TI_PRIM_CLIPPED", -+ "T76x_TI_LEVEL0", -+ "T76x_TI_LEVEL1", -+ "T76x_TI_LEVEL2", -+ "T76x_TI_LEVEL3", -+ "T76x_TI_LEVEL4", -+ "T76x_TI_LEVEL5", -+ "T76x_TI_LEVEL6", -+ "T76x_TI_LEVEL7", -+ "T76x_TI_COMMAND_1", -+ "T76x_TI_COMMAND_2", -+ "T76x_TI_COMMAND_3", -+ "T76x_TI_COMMAND_4", -+ "T76x_TI_COMMAND_5_7", -+ "T76x_TI_COMMAND_8_15", -+ "T76x_TI_COMMAND_16_63", -+ "T76x_TI_COMMAND_64", -+ "T76x_TI_COMPRESS_IN", -+ "T76x_TI_COMPRESS_OUT", -+ "T76x_TI_COMPRESS_FLUSH", -+ "T76x_TI_TIMESTAMPS", -+ "T76x_TI_PCACHE_HIT", -+ "T76x_TI_PCACHE_MISS", -+ "T76x_TI_PCACHE_LINE", -+ "T76x_TI_PCACHE_STALL", -+ "T76x_TI_WRBUF_HIT", -+ "T76x_TI_WRBUF_MISS", -+ "T76x_TI_WRBUF_LINE", -+ "T76x_TI_WRBUF_PARTIAL", -+ "T76x_TI_WRBUF_STALL", -+ "T76x_TI_ACTIVE", -+ "T76x_TI_LOADING_DESC", -+ "T76x_TI_INDEX_WAIT", -+ "T76x_TI_INDEX_RANGE_WAIT", -+ "T76x_TI_VERTEX_WAIT", -+ "T76x_TI_PCACHE_WAIT", -+ "T76x_TI_WRBUF_WAIT", -+ "T76x_TI_BUS_READ", -+ "T76x_TI_BUS_WRITE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T76x_TI_UTLB_HIT", -+ "T76x_TI_UTLB_NEW_MISS", -+ "T76x_TI_UTLB_REPLAY_FULL", -+ "T76x_TI_UTLB_REPLAY_MISS", -+ "T76x_TI_UTLB_STALL", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T76x_FRAG_ACTIVE", -+ "T76x_FRAG_PRIMITIVES", -+ "T76x_FRAG_PRIMITIVES_DROPPED", -+ "T76x_FRAG_CYCLES_DESC", -+ "T76x_FRAG_CYCLES_FPKQ_ACTIVE", -+ "T76x_FRAG_CYCLES_VERT", -+ "T76x_FRAG_CYCLES_TRISETUP", -+ "T76x_FRAG_CYCLES_EZS_ACTIVE", -+ "T76x_FRAG_THREADS", -+ "T76x_FRAG_DUMMY_THREADS", -+ "T76x_FRAG_QUADS_RAST", -+ "T76x_FRAG_QUADS_EZS_TEST", -+ "T76x_FRAG_QUADS_EZS_KILLED", -+ "T76x_FRAG_THREADS_LZS_TEST", -+ "T76x_FRAG_THREADS_LZS_KILLED", -+ "T76x_FRAG_CYCLES_NO_TILE", -+ "T76x_FRAG_NUM_TILES", -+ "T76x_FRAG_TRANS_ELIM", -+ "T76x_COMPUTE_ACTIVE", -+ "T76x_COMPUTE_TASKS", -+ "T76x_COMPUTE_THREADS", -+ "T76x_COMPUTE_CYCLES_DESC", -+ "T76x_TRIPIPE_ACTIVE", -+ "T76x_ARITH_WORDS", -+ "T76x_ARITH_CYCLES_REG", -+ "T76x_ARITH_CYCLES_L0", -+ "T76x_ARITH_FRAG_DEPEND", -+ "T76x_LS_WORDS", -+ "T76x_LS_ISSUES", -+ "T76x_LS_REISSUE_ATTR", -+ "T76x_LS_REISSUES_VARY", -+ "T76x_LS_VARY_RV_MISS", -+ "T76x_LS_VARY_RV_HIT", -+ "T76x_LS_NO_UNPARK", -+ "T76x_TEX_WORDS", -+ "T76x_TEX_BUBBLES", -+ "T76x_TEX_WORDS_L0", -+ "T76x_TEX_WORDS_DESC", -+ "T76x_TEX_ISSUES", -+ "T76x_TEX_RECIRC_FMISS", -+ "T76x_TEX_RECIRC_DESC", -+ "T76x_TEX_RECIRC_MULTI", -+ "T76x_TEX_RECIRC_PMISS", -+ "T76x_TEX_RECIRC_CONF", -+ "T76x_LSC_READ_HITS", -+ "T76x_LSC_READ_OP", -+ "T76x_LSC_WRITE_HITS", -+ "T76x_LSC_WRITE_OP", -+ "T76x_LSC_ATOMIC_HITS", -+ "T76x_LSC_ATOMIC_OP", -+ "T76x_LSC_LINE_FETCHES", -+ "T76x_LSC_DIRTY_LINE", -+ "T76x_LSC_SNOOPS", -+ "T76x_AXI_TLB_STALL", -+ "T76x_AXI_TLB_MISS", -+ "T76x_AXI_TLB_TRANSACTION", -+ "T76x_LS_TLB_MISS", -+ "T76x_LS_TLB_HIT", -+ "T76x_AXI_BEATS_READ", -+ "T76x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T76x_MMU_HIT", -+ "T76x_MMU_NEW_MISS", -+ "T76x_MMU_REPLAY_FULL", -+ "T76x_MMU_REPLAY_MISS", -+ "T76x_MMU_TABLE_WALK", -+ "T76x_MMU_REQUESTS", -+ "", -+ "", -+ "T76x_UTLB_HIT", -+ "T76x_UTLB_NEW_MISS", -+ "T76x_UTLB_REPLAY_FULL", -+ "T76x_UTLB_REPLAY_MISS", -+ "T76x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T76x_L2_EXT_WRITE_BEATS", -+ "T76x_L2_EXT_READ_BEATS", -+ "T76x_L2_ANY_LOOKUP", -+ "T76x_L2_READ_LOOKUP", -+ "T76x_L2_SREAD_LOOKUP", -+ "T76x_L2_READ_REPLAY", -+ "T76x_L2_READ_SNOOP", -+ "T76x_L2_READ_HIT", -+ "T76x_L2_CLEAN_MISS", -+ "T76x_L2_WRITE_LOOKUP", -+ "T76x_L2_SWRITE_LOOKUP", -+ "T76x_L2_WRITE_REPLAY", -+ "T76x_L2_WRITE_SNOOP", -+ "T76x_L2_WRITE_HIT", -+ "T76x_L2_EXT_READ_FULL", -+ "", -+ "T76x_L2_EXT_WRITE_FULL", -+ "T76x_L2_EXT_R_W_HAZARD", -+ "T76x_L2_EXT_READ", -+ "T76x_L2_EXT_READ_LINE", -+ "T76x_L2_EXT_WRITE", -+ "T76x_L2_EXT_WRITE_LINE", -+ "T76x_L2_EXT_WRITE_SMALL", -+ "T76x_L2_EXT_BARRIER", -+ "T76x_L2_EXT_AR_STALL", -+ "T76x_L2_EXT_R_BUF_FULL", -+ "T76x_L2_EXT_RD_BUF_FULL", -+ "T76x_L2_EXT_R_RAW", -+ "T76x_L2_EXT_W_STALL", -+ "T76x_L2_EXT_W_BUF_FULL", -+ "T76x_L2_EXT_R_BUF_FULL", -+ "T76x_L2_TAG_HAZARD", -+ "T76x_L2_SNOOP_FULL", -+ "T76x_L2_REPLAY_FULL" -+}; -+ -+static const char * const hardware_counters_mali_t82x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T82x_MESSAGES_SENT", -+ "T82x_MESSAGES_RECEIVED", -+ "T82x_GPU_ACTIVE", -+ "T82x_IRQ_ACTIVE", -+ "T82x_JS0_JOBS", -+ "T82x_JS0_TASKS", -+ "T82x_JS0_ACTIVE", -+ "", -+ "T82x_JS0_WAIT_READ", -+ "T82x_JS0_WAIT_ISSUE", -+ "T82x_JS0_WAIT_DEPEND", -+ "T82x_JS0_WAIT_FINISH", -+ "T82x_JS1_JOBS", -+ "T82x_JS1_TASKS", -+ "T82x_JS1_ACTIVE", -+ "", -+ "T82x_JS1_WAIT_READ", -+ "T82x_JS1_WAIT_ISSUE", -+ "T82x_JS1_WAIT_DEPEND", -+ "T82x_JS1_WAIT_FINISH", -+ "T82x_JS2_JOBS", -+ "T82x_JS2_TASKS", -+ "T82x_JS2_ACTIVE", -+ "", -+ "T82x_JS2_WAIT_READ", -+ "T82x_JS2_WAIT_ISSUE", -+ "T82x_JS2_WAIT_DEPEND", -+ "T82x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T82x_TI_JOBS_PROCESSED", -+ "T82x_TI_TRIANGLES", -+ "T82x_TI_QUADS", -+ "T82x_TI_POLYGONS", -+ "T82x_TI_POINTS", -+ "T82x_TI_LINES", -+ "T82x_TI_FRONT_FACING", -+ "T82x_TI_BACK_FACING", -+ "T82x_TI_PRIM_VISIBLE", -+ "T82x_TI_PRIM_CULLED", -+ "T82x_TI_PRIM_CLIPPED", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T82x_TI_ACTIVE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T82x_FRAG_ACTIVE", -+ "T82x_FRAG_PRIMITIVES", -+ "T82x_FRAG_PRIMITIVES_DROPPED", -+ "T82x_FRAG_CYCLES_DESC", -+ "T82x_FRAG_CYCLES_FPKQ_ACTIVE", -+ "T82x_FRAG_CYCLES_VERT", -+ "T82x_FRAG_CYCLES_TRISETUP", -+ "T82x_FRAG_CYCLES_EZS_ACTIVE", -+ "T82x_FRAG_THREADS", -+ "T82x_FRAG_DUMMY_THREADS", -+ "T82x_FRAG_QUADS_RAST", -+ "T82x_FRAG_QUADS_EZS_TEST", -+ "T82x_FRAG_QUADS_EZS_KILLED", -+ "T82x_FRAG_THREADS_LZS_TEST", -+ "T82x_FRAG_THREADS_LZS_KILLED", -+ "T82x_FRAG_CYCLES_NO_TILE", -+ "T82x_FRAG_NUM_TILES", -+ "T82x_FRAG_TRANS_ELIM", -+ "T82x_COMPUTE_ACTIVE", -+ "T82x_COMPUTE_TASKS", -+ "T82x_COMPUTE_THREADS", -+ "T82x_COMPUTE_CYCLES_DESC", -+ "T82x_TRIPIPE_ACTIVE", -+ "T82x_ARITH_WORDS", -+ "T82x_ARITH_CYCLES_REG", -+ "T82x_ARITH_CYCLES_L0", -+ "T82x_ARITH_FRAG_DEPEND", -+ "T82x_LS_WORDS", -+ "T82x_LS_ISSUES", -+ "T82x_LS_REISSUE_ATTR", -+ "T82x_LS_REISSUES_VARY", -+ "T82x_LS_VARY_RV_MISS", -+ "T82x_LS_VARY_RV_HIT", -+ "T82x_LS_NO_UNPARK", -+ "T82x_TEX_WORDS", -+ "T82x_TEX_BUBBLES", -+ "T82x_TEX_WORDS_L0", -+ "T82x_TEX_WORDS_DESC", -+ "T82x_TEX_ISSUES", -+ "T82x_TEX_RECIRC_FMISS", -+ "T82x_TEX_RECIRC_DESC", -+ "T82x_TEX_RECIRC_MULTI", -+ "T82x_TEX_RECIRC_PMISS", -+ "T82x_TEX_RECIRC_CONF", -+ "T82x_LSC_READ_HITS", -+ "T82x_LSC_READ_OP", -+ "T82x_LSC_WRITE_HITS", -+ "T82x_LSC_WRITE_OP", -+ "T82x_LSC_ATOMIC_HITS", -+ "T82x_LSC_ATOMIC_OP", -+ "T82x_LSC_LINE_FETCHES", -+ "T82x_LSC_DIRTY_LINE", -+ "T82x_LSC_SNOOPS", -+ "T82x_AXI_TLB_STALL", -+ "T82x_AXI_TLB_MISS", -+ "T82x_AXI_TLB_TRANSACTION", -+ "T82x_LS_TLB_MISS", -+ "T82x_LS_TLB_HIT", -+ "T82x_AXI_BEATS_READ", -+ "T82x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T82x_MMU_HIT", -+ "T82x_MMU_NEW_MISS", -+ "T82x_MMU_REPLAY_FULL", -+ "T82x_MMU_REPLAY_MISS", -+ "T82x_MMU_TABLE_WALK", -+ "T82x_MMU_REQUESTS", -+ "", -+ "", -+ "T82x_UTLB_HIT", -+ "T82x_UTLB_NEW_MISS", -+ "T82x_UTLB_REPLAY_FULL", -+ "T82x_UTLB_REPLAY_MISS", -+ "T82x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T82x_L2_EXT_WRITE_BEATS", -+ "T82x_L2_EXT_READ_BEATS", -+ "T82x_L2_ANY_LOOKUP", -+ "T82x_L2_READ_LOOKUP", -+ "T82x_L2_SREAD_LOOKUP", -+ "T82x_L2_READ_REPLAY", -+ "T82x_L2_READ_SNOOP", -+ "T82x_L2_READ_HIT", -+ "T82x_L2_CLEAN_MISS", -+ "T82x_L2_WRITE_LOOKUP", -+ "T82x_L2_SWRITE_LOOKUP", -+ "T82x_L2_WRITE_REPLAY", -+ "T82x_L2_WRITE_SNOOP", -+ "T82x_L2_WRITE_HIT", -+ "T82x_L2_EXT_READ_FULL", -+ "", -+ "T82x_L2_EXT_WRITE_FULL", -+ "T82x_L2_EXT_R_W_HAZARD", -+ "T82x_L2_EXT_READ", -+ "T82x_L2_EXT_READ_LINE", -+ "T82x_L2_EXT_WRITE", -+ "T82x_L2_EXT_WRITE_LINE", -+ "T82x_L2_EXT_WRITE_SMALL", -+ "T82x_L2_EXT_BARRIER", -+ "T82x_L2_EXT_AR_STALL", -+ "T82x_L2_EXT_R_BUF_FULL", -+ "T82x_L2_EXT_RD_BUF_FULL", -+ "T82x_L2_EXT_R_RAW", -+ "T82x_L2_EXT_W_STALL", -+ "T82x_L2_EXT_W_BUF_FULL", -+ "T82x_L2_EXT_R_BUF_FULL", -+ "T82x_L2_TAG_HAZARD", -+ "T82x_L2_SNOOP_FULL", -+ "T82x_L2_REPLAY_FULL" -+}; -+ -+static const char * const hardware_counters_mali_t83x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T83x_MESSAGES_SENT", -+ "T83x_MESSAGES_RECEIVED", -+ "T83x_GPU_ACTIVE", -+ "T83x_IRQ_ACTIVE", -+ "T83x_JS0_JOBS", -+ "T83x_JS0_TASKS", -+ "T83x_JS0_ACTIVE", -+ "", -+ "T83x_JS0_WAIT_READ", -+ "T83x_JS0_WAIT_ISSUE", -+ "T83x_JS0_WAIT_DEPEND", -+ "T83x_JS0_WAIT_FINISH", -+ "T83x_JS1_JOBS", -+ "T83x_JS1_TASKS", -+ "T83x_JS1_ACTIVE", -+ "", -+ "T83x_JS1_WAIT_READ", -+ "T83x_JS1_WAIT_ISSUE", -+ "T83x_JS1_WAIT_DEPEND", -+ "T83x_JS1_WAIT_FINISH", -+ "T83x_JS2_JOBS", -+ "T83x_JS2_TASKS", -+ "T83x_JS2_ACTIVE", -+ "", -+ "T83x_JS2_WAIT_READ", -+ "T83x_JS2_WAIT_ISSUE", -+ "T83x_JS2_WAIT_DEPEND", -+ "T83x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T83x_TI_JOBS_PROCESSED", -+ "T83x_TI_TRIANGLES", -+ "T83x_TI_QUADS", -+ "T83x_TI_POLYGONS", -+ "T83x_TI_POINTS", -+ "T83x_TI_LINES", -+ "T83x_TI_FRONT_FACING", -+ "T83x_TI_BACK_FACING", -+ "T83x_TI_PRIM_VISIBLE", -+ "T83x_TI_PRIM_CULLED", -+ "T83x_TI_PRIM_CLIPPED", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T83x_TI_ACTIVE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T83x_FRAG_ACTIVE", -+ "T83x_FRAG_PRIMITIVES", -+ "T83x_FRAG_PRIMITIVES_DROPPED", -+ "T83x_FRAG_CYCLES_DESC", -+ "T83x_FRAG_CYCLES_FPKQ_ACTIVE", -+ "T83x_FRAG_CYCLES_VERT", -+ "T83x_FRAG_CYCLES_TRISETUP", -+ "T83x_FRAG_CYCLES_EZS_ACTIVE", -+ "T83x_FRAG_THREADS", -+ "T83x_FRAG_DUMMY_THREADS", -+ "T83x_FRAG_QUADS_RAST", -+ "T83x_FRAG_QUADS_EZS_TEST", -+ "T83x_FRAG_QUADS_EZS_KILLED", -+ "T83x_FRAG_THREADS_LZS_TEST", -+ "T83x_FRAG_THREADS_LZS_KILLED", -+ "T83x_FRAG_CYCLES_NO_TILE", -+ "T83x_FRAG_NUM_TILES", -+ "T83x_FRAG_TRANS_ELIM", -+ "T83x_COMPUTE_ACTIVE", -+ "T83x_COMPUTE_TASKS", -+ "T83x_COMPUTE_THREADS", -+ "T83x_COMPUTE_CYCLES_DESC", -+ "T83x_TRIPIPE_ACTIVE", -+ "T83x_ARITH_WORDS", -+ "T83x_ARITH_CYCLES_REG", -+ "T83x_ARITH_CYCLES_L0", -+ "T83x_ARITH_FRAG_DEPEND", -+ "T83x_LS_WORDS", -+ "T83x_LS_ISSUES", -+ "T83x_LS_REISSUE_ATTR", -+ "T83x_LS_REISSUES_VARY", -+ "T83x_LS_VARY_RV_MISS", -+ "T83x_LS_VARY_RV_HIT", -+ "T83x_LS_NO_UNPARK", -+ "T83x_TEX_WORDS", -+ "T83x_TEX_BUBBLES", -+ "T83x_TEX_WORDS_L0", -+ "T83x_TEX_WORDS_DESC", -+ "T83x_TEX_ISSUES", -+ "T83x_TEX_RECIRC_FMISS", -+ "T83x_TEX_RECIRC_DESC", -+ "T83x_TEX_RECIRC_MULTI", -+ "T83x_TEX_RECIRC_PMISS", -+ "T83x_TEX_RECIRC_CONF", -+ "T83x_LSC_READ_HITS", -+ "T83x_LSC_READ_OP", -+ "T83x_LSC_WRITE_HITS", -+ "T83x_LSC_WRITE_OP", -+ "T83x_LSC_ATOMIC_HITS", -+ "T83x_LSC_ATOMIC_OP", -+ "T83x_LSC_LINE_FETCHES", -+ "T83x_LSC_DIRTY_LINE", -+ "T83x_LSC_SNOOPS", -+ "T83x_AXI_TLB_STALL", -+ "T83x_AXI_TLB_MISS", -+ "T83x_AXI_TLB_TRANSACTION", -+ "T83x_LS_TLB_MISS", -+ "T83x_LS_TLB_HIT", -+ "T83x_AXI_BEATS_READ", -+ "T83x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T83x_MMU_HIT", -+ "T83x_MMU_NEW_MISS", -+ "T83x_MMU_REPLAY_FULL", -+ "T83x_MMU_REPLAY_MISS", -+ "T83x_MMU_TABLE_WALK", -+ "T83x_MMU_REQUESTS", -+ "", -+ "", -+ "T83x_UTLB_HIT", -+ "T83x_UTLB_NEW_MISS", -+ "T83x_UTLB_REPLAY_FULL", -+ "T83x_UTLB_REPLAY_MISS", -+ "T83x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T83x_L2_EXT_WRITE_BEATS", -+ "T83x_L2_EXT_READ_BEATS", -+ "T83x_L2_ANY_LOOKUP", -+ "T83x_L2_READ_LOOKUP", -+ "T83x_L2_SREAD_LOOKUP", -+ "T83x_L2_READ_REPLAY", -+ "T83x_L2_READ_SNOOP", -+ "T83x_L2_READ_HIT", -+ "T83x_L2_CLEAN_MISS", -+ "T83x_L2_WRITE_LOOKUP", -+ "T83x_L2_SWRITE_LOOKUP", -+ "T83x_L2_WRITE_REPLAY", -+ "T83x_L2_WRITE_SNOOP", -+ "T83x_L2_WRITE_HIT", -+ "T83x_L2_EXT_READ_FULL", -+ "", -+ "T83x_L2_EXT_WRITE_FULL", -+ "T83x_L2_EXT_R_W_HAZARD", -+ "T83x_L2_EXT_READ", -+ "T83x_L2_EXT_READ_LINE", -+ "T83x_L2_EXT_WRITE", -+ "T83x_L2_EXT_WRITE_LINE", -+ "T83x_L2_EXT_WRITE_SMALL", -+ "T83x_L2_EXT_BARRIER", -+ "T83x_L2_EXT_AR_STALL", -+ "T83x_L2_EXT_R_BUF_FULL", -+ "T83x_L2_EXT_RD_BUF_FULL", -+ "T83x_L2_EXT_R_RAW", -+ "T83x_L2_EXT_W_STALL", -+ "T83x_L2_EXT_W_BUF_FULL", -+ "T83x_L2_EXT_R_BUF_FULL", -+ "T83x_L2_TAG_HAZARD", -+ "T83x_L2_SNOOP_FULL", -+ "T83x_L2_REPLAY_FULL" -+}; -+ -+static const char * const hardware_counters_mali_t86x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T86x_MESSAGES_SENT", -+ "T86x_MESSAGES_RECEIVED", -+ "T86x_GPU_ACTIVE", -+ "T86x_IRQ_ACTIVE", -+ "T86x_JS0_JOBS", -+ "T86x_JS0_TASKS", -+ "T86x_JS0_ACTIVE", -+ "", -+ "T86x_JS0_WAIT_READ", -+ "T86x_JS0_WAIT_ISSUE", -+ "T86x_JS0_WAIT_DEPEND", -+ "T86x_JS0_WAIT_FINISH", -+ "T86x_JS1_JOBS", -+ "T86x_JS1_TASKS", -+ "T86x_JS1_ACTIVE", -+ "", -+ "T86x_JS1_WAIT_READ", -+ "T86x_JS1_WAIT_ISSUE", -+ "T86x_JS1_WAIT_DEPEND", -+ "T86x_JS1_WAIT_FINISH", -+ "T86x_JS2_JOBS", -+ "T86x_JS2_TASKS", -+ "T86x_JS2_ACTIVE", -+ "", -+ "T86x_JS2_WAIT_READ", -+ "T86x_JS2_WAIT_ISSUE", -+ "T86x_JS2_WAIT_DEPEND", -+ "T86x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T86x_TI_JOBS_PROCESSED", -+ "T86x_TI_TRIANGLES", -+ "T86x_TI_QUADS", -+ "T86x_TI_POLYGONS", -+ "T86x_TI_POINTS", -+ "T86x_TI_LINES", -+ "T86x_TI_VCACHE_HIT", -+ "T86x_TI_VCACHE_MISS", -+ "T86x_TI_FRONT_FACING", -+ "T86x_TI_BACK_FACING", -+ "T86x_TI_PRIM_VISIBLE", -+ "T86x_TI_PRIM_CULLED", -+ "T86x_TI_PRIM_CLIPPED", -+ "T86x_TI_LEVEL0", -+ "T86x_TI_LEVEL1", -+ "T86x_TI_LEVEL2", -+ "T86x_TI_LEVEL3", -+ "T86x_TI_LEVEL4", -+ "T86x_TI_LEVEL5", -+ "T86x_TI_LEVEL6", -+ "T86x_TI_LEVEL7", -+ "T86x_TI_COMMAND_1", -+ "T86x_TI_COMMAND_2", -+ "T86x_TI_COMMAND_3", -+ "T86x_TI_COMMAND_4", -+ "T86x_TI_COMMAND_5_7", -+ "T86x_TI_COMMAND_8_15", -+ "T86x_TI_COMMAND_16_63", -+ "T86x_TI_COMMAND_64", -+ "T86x_TI_COMPRESS_IN", -+ "T86x_TI_COMPRESS_OUT", -+ "T86x_TI_COMPRESS_FLUSH", -+ "T86x_TI_TIMESTAMPS", -+ "T86x_TI_PCACHE_HIT", -+ "T86x_TI_PCACHE_MISS", -+ "T86x_TI_PCACHE_LINE", -+ "T86x_TI_PCACHE_STALL", -+ "T86x_TI_WRBUF_HIT", -+ "T86x_TI_WRBUF_MISS", -+ "T86x_TI_WRBUF_LINE", -+ "T86x_TI_WRBUF_PARTIAL", -+ "T86x_TI_WRBUF_STALL", -+ "T86x_TI_ACTIVE", -+ "T86x_TI_LOADING_DESC", -+ "T86x_TI_INDEX_WAIT", -+ "T86x_TI_INDEX_RANGE_WAIT", -+ "T86x_TI_VERTEX_WAIT", -+ "T86x_TI_PCACHE_WAIT", -+ "T86x_TI_WRBUF_WAIT", -+ "T86x_TI_BUS_READ", -+ "T86x_TI_BUS_WRITE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T86x_TI_UTLB_HIT", -+ "T86x_TI_UTLB_NEW_MISS", -+ "T86x_TI_UTLB_REPLAY_FULL", -+ "T86x_TI_UTLB_REPLAY_MISS", -+ "T86x_TI_UTLB_STALL", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T86x_FRAG_ACTIVE", -+ "T86x_FRAG_PRIMITIVES", -+ "T86x_FRAG_PRIMITIVES_DROPPED", -+ "T86x_FRAG_CYCLES_DESC", -+ "T86x_FRAG_CYCLES_FPKQ_ACTIVE", -+ "T86x_FRAG_CYCLES_VERT", -+ "T86x_FRAG_CYCLES_TRISETUP", -+ "T86x_FRAG_CYCLES_EZS_ACTIVE", -+ "T86x_FRAG_THREADS", -+ "T86x_FRAG_DUMMY_THREADS", -+ "T86x_FRAG_QUADS_RAST", -+ "T86x_FRAG_QUADS_EZS_TEST", -+ "T86x_FRAG_QUADS_EZS_KILLED", -+ "T86x_FRAG_THREADS_LZS_TEST", -+ "T86x_FRAG_THREADS_LZS_KILLED", -+ "T86x_FRAG_CYCLES_NO_TILE", -+ "T86x_FRAG_NUM_TILES", -+ "T86x_FRAG_TRANS_ELIM", -+ "T86x_COMPUTE_ACTIVE", -+ "T86x_COMPUTE_TASKS", -+ "T86x_COMPUTE_THREADS", -+ "T86x_COMPUTE_CYCLES_DESC", -+ "T86x_TRIPIPE_ACTIVE", -+ "T86x_ARITH_WORDS", -+ "T86x_ARITH_CYCLES_REG", -+ "T86x_ARITH_CYCLES_L0", -+ "T86x_ARITH_FRAG_DEPEND", -+ "T86x_LS_WORDS", -+ "T86x_LS_ISSUES", -+ "T86x_LS_REISSUE_ATTR", -+ "T86x_LS_REISSUES_VARY", -+ "T86x_LS_VARY_RV_MISS", -+ "T86x_LS_VARY_RV_HIT", -+ "T86x_LS_NO_UNPARK", -+ "T86x_TEX_WORDS", -+ "T86x_TEX_BUBBLES", -+ "T86x_TEX_WORDS_L0", -+ "T86x_TEX_WORDS_DESC", -+ "T86x_TEX_ISSUES", -+ "T86x_TEX_RECIRC_FMISS", -+ "T86x_TEX_RECIRC_DESC", -+ "T86x_TEX_RECIRC_MULTI", -+ "T86x_TEX_RECIRC_PMISS", -+ "T86x_TEX_RECIRC_CONF", -+ "T86x_LSC_READ_HITS", -+ "T86x_LSC_READ_OP", -+ "T86x_LSC_WRITE_HITS", -+ "T86x_LSC_WRITE_OP", -+ "T86x_LSC_ATOMIC_HITS", -+ "T86x_LSC_ATOMIC_OP", -+ "T86x_LSC_LINE_FETCHES", -+ "T86x_LSC_DIRTY_LINE", -+ "T86x_LSC_SNOOPS", -+ "T86x_AXI_TLB_STALL", -+ "T86x_AXI_TLB_MISS", -+ "T86x_AXI_TLB_TRANSACTION", -+ "T86x_LS_TLB_MISS", -+ "T86x_LS_TLB_HIT", -+ "T86x_AXI_BEATS_READ", -+ "T86x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T86x_MMU_HIT", -+ "T86x_MMU_NEW_MISS", -+ "T86x_MMU_REPLAY_FULL", -+ "T86x_MMU_REPLAY_MISS", -+ "T86x_MMU_TABLE_WALK", -+ "T86x_MMU_REQUESTS", -+ "", -+ "", -+ "T86x_UTLB_HIT", -+ "T86x_UTLB_NEW_MISS", -+ "T86x_UTLB_REPLAY_FULL", -+ "T86x_UTLB_REPLAY_MISS", -+ "T86x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T86x_L2_EXT_WRITE_BEATS", -+ "T86x_L2_EXT_READ_BEATS", -+ "T86x_L2_ANY_LOOKUP", -+ "T86x_L2_READ_LOOKUP", -+ "T86x_L2_SREAD_LOOKUP", -+ "T86x_L2_READ_REPLAY", -+ "T86x_L2_READ_SNOOP", -+ "T86x_L2_READ_HIT", -+ "T86x_L2_CLEAN_MISS", -+ "T86x_L2_WRITE_LOOKUP", -+ "T86x_L2_SWRITE_LOOKUP", -+ "T86x_L2_WRITE_REPLAY", -+ "T86x_L2_WRITE_SNOOP", -+ "T86x_L2_WRITE_HIT", -+ "T86x_L2_EXT_READ_FULL", -+ "", -+ "T86x_L2_EXT_WRITE_FULL", -+ "T86x_L2_EXT_R_W_HAZARD", -+ "T86x_L2_EXT_READ", -+ "T86x_L2_EXT_READ_LINE", -+ "T86x_L2_EXT_WRITE", -+ "T86x_L2_EXT_WRITE_LINE", -+ "T86x_L2_EXT_WRITE_SMALL", -+ "T86x_L2_EXT_BARRIER", -+ "T86x_L2_EXT_AR_STALL", -+ "T86x_L2_EXT_R_BUF_FULL", -+ "T86x_L2_EXT_RD_BUF_FULL", -+ "T86x_L2_EXT_R_RAW", -+ "T86x_L2_EXT_W_STALL", -+ "T86x_L2_EXT_W_BUF_FULL", -+ "T86x_L2_EXT_R_BUF_FULL", -+ "T86x_L2_TAG_HAZARD", -+ "T86x_L2_SNOOP_FULL", -+ "T86x_L2_REPLAY_FULL" -+}; -+ -+static const char * const hardware_counters_mali_t88x[] = { -+ /* Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "T88x_MESSAGES_SENT", -+ "T88x_MESSAGES_RECEIVED", -+ "T88x_GPU_ACTIVE", -+ "T88x_IRQ_ACTIVE", -+ "T88x_JS0_JOBS", -+ "T88x_JS0_TASKS", -+ "T88x_JS0_ACTIVE", -+ "", -+ "T88x_JS0_WAIT_READ", -+ "T88x_JS0_WAIT_ISSUE", -+ "T88x_JS0_WAIT_DEPEND", -+ "T88x_JS0_WAIT_FINISH", -+ "T88x_JS1_JOBS", -+ "T88x_JS1_TASKS", -+ "T88x_JS1_ACTIVE", -+ "", -+ "T88x_JS1_WAIT_READ", -+ "T88x_JS1_WAIT_ISSUE", -+ "T88x_JS1_WAIT_DEPEND", -+ "T88x_JS1_WAIT_FINISH", -+ "T88x_JS2_JOBS", -+ "T88x_JS2_TASKS", -+ "T88x_JS2_ACTIVE", -+ "", -+ "T88x_JS2_WAIT_READ", -+ "T88x_JS2_WAIT_ISSUE", -+ "T88x_JS2_WAIT_DEPEND", -+ "T88x_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /*Tiler */ -+ "", -+ "", -+ "", -+ "T88x_TI_JOBS_PROCESSED", -+ "T88x_TI_TRIANGLES", -+ "T88x_TI_QUADS", -+ "T88x_TI_POLYGONS", -+ "T88x_TI_POINTS", -+ "T88x_TI_LINES", -+ "T88x_TI_VCACHE_HIT", -+ "T88x_TI_VCACHE_MISS", -+ "T88x_TI_FRONT_FACING", -+ "T88x_TI_BACK_FACING", -+ "T88x_TI_PRIM_VISIBLE", -+ "T88x_TI_PRIM_CULLED", -+ "T88x_TI_PRIM_CLIPPED", -+ "T88x_TI_LEVEL0", -+ "T88x_TI_LEVEL1", -+ "T88x_TI_LEVEL2", -+ "T88x_TI_LEVEL3", -+ "T88x_TI_LEVEL4", -+ "T88x_TI_LEVEL5", -+ "T88x_TI_LEVEL6", -+ "T88x_TI_LEVEL7", -+ "T88x_TI_COMMAND_1", -+ "T88x_TI_COMMAND_2", -+ "T88x_TI_COMMAND_3", -+ "T88x_TI_COMMAND_4", -+ "T88x_TI_COMMAND_5_7", -+ "T88x_TI_COMMAND_8_15", -+ "T88x_TI_COMMAND_16_63", -+ "T88x_TI_COMMAND_64", -+ "T88x_TI_COMPRESS_IN", -+ "T88x_TI_COMPRESS_OUT", -+ "T88x_TI_COMPRESS_FLUSH", -+ "T88x_TI_TIMESTAMPS", -+ "T88x_TI_PCACHE_HIT", -+ "T88x_TI_PCACHE_MISS", -+ "T88x_TI_PCACHE_LINE", -+ "T88x_TI_PCACHE_STALL", -+ "T88x_TI_WRBUF_HIT", -+ "T88x_TI_WRBUF_MISS", -+ "T88x_TI_WRBUF_LINE", -+ "T88x_TI_WRBUF_PARTIAL", -+ "T88x_TI_WRBUF_STALL", -+ "T88x_TI_ACTIVE", -+ "T88x_TI_LOADING_DESC", -+ "T88x_TI_INDEX_WAIT", -+ "T88x_TI_INDEX_RANGE_WAIT", -+ "T88x_TI_VERTEX_WAIT", -+ "T88x_TI_PCACHE_WAIT", -+ "T88x_TI_WRBUF_WAIT", -+ "T88x_TI_BUS_READ", -+ "T88x_TI_BUS_WRITE", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T88x_TI_UTLB_HIT", -+ "T88x_TI_UTLB_NEW_MISS", -+ "T88x_TI_UTLB_REPLAY_FULL", -+ "T88x_TI_UTLB_REPLAY_MISS", -+ "T88x_TI_UTLB_STALL", -+ -+ /* Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "T88x_FRAG_ACTIVE", -+ "T88x_FRAG_PRIMITIVES", -+ "T88x_FRAG_PRIMITIVES_DROPPED", -+ "T88x_FRAG_CYCLES_DESC", -+ "T88x_FRAG_CYCLES_FPKQ_ACTIVE", -+ "T88x_FRAG_CYCLES_VERT", -+ "T88x_FRAG_CYCLES_TRISETUP", -+ "T88x_FRAG_CYCLES_EZS_ACTIVE", -+ "T88x_FRAG_THREADS", -+ "T88x_FRAG_DUMMY_THREADS", -+ "T88x_FRAG_QUADS_RAST", -+ "T88x_FRAG_QUADS_EZS_TEST", -+ "T88x_FRAG_QUADS_EZS_KILLED", -+ "T88x_FRAG_THREADS_LZS_TEST", -+ "T88x_FRAG_THREADS_LZS_KILLED", -+ "T88x_FRAG_CYCLES_NO_TILE", -+ "T88x_FRAG_NUM_TILES", -+ "T88x_FRAG_TRANS_ELIM", -+ "T88x_COMPUTE_ACTIVE", -+ "T88x_COMPUTE_TASKS", -+ "T88x_COMPUTE_THREADS", -+ "T88x_COMPUTE_CYCLES_DESC", -+ "T88x_TRIPIPE_ACTIVE", -+ "T88x_ARITH_WORDS", -+ "T88x_ARITH_CYCLES_REG", -+ "T88x_ARITH_CYCLES_L0", -+ "T88x_ARITH_FRAG_DEPEND", -+ "T88x_LS_WORDS", -+ "T88x_LS_ISSUES", -+ "T88x_LS_REISSUE_ATTR", -+ "T88x_LS_REISSUES_VARY", -+ "T88x_LS_VARY_RV_MISS", -+ "T88x_LS_VARY_RV_HIT", -+ "T88x_LS_NO_UNPARK", -+ "T88x_TEX_WORDS", -+ "T88x_TEX_BUBBLES", -+ "T88x_TEX_WORDS_L0", -+ "T88x_TEX_WORDS_DESC", -+ "T88x_TEX_ISSUES", -+ "T88x_TEX_RECIRC_FMISS", -+ "T88x_TEX_RECIRC_DESC", -+ "T88x_TEX_RECIRC_MULTI", -+ "T88x_TEX_RECIRC_PMISS", -+ "T88x_TEX_RECIRC_CONF", -+ "T88x_LSC_READ_HITS", -+ "T88x_LSC_READ_OP", -+ "T88x_LSC_WRITE_HITS", -+ "T88x_LSC_WRITE_OP", -+ "T88x_LSC_ATOMIC_HITS", -+ "T88x_LSC_ATOMIC_OP", -+ "T88x_LSC_LINE_FETCHES", -+ "T88x_LSC_DIRTY_LINE", -+ "T88x_LSC_SNOOPS", -+ "T88x_AXI_TLB_STALL", -+ "T88x_AXI_TLB_MISS", -+ "T88x_AXI_TLB_TRANSACTION", -+ "T88x_LS_TLB_MISS", -+ "T88x_LS_TLB_HIT", -+ "T88x_AXI_BEATS_READ", -+ "T88x_AXI_BEATS_WRITTEN", -+ -+ /*L2 and MMU */ -+ "", -+ "", -+ "", -+ "", -+ "T88x_MMU_HIT", -+ "T88x_MMU_NEW_MISS", -+ "T88x_MMU_REPLAY_FULL", -+ "T88x_MMU_REPLAY_MISS", -+ "T88x_MMU_TABLE_WALK", -+ "T88x_MMU_REQUESTS", -+ "", -+ "", -+ "T88x_UTLB_HIT", -+ "T88x_UTLB_NEW_MISS", -+ "T88x_UTLB_REPLAY_FULL", -+ "T88x_UTLB_REPLAY_MISS", -+ "T88x_UTLB_STALL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "T88x_L2_EXT_WRITE_BEATS", -+ "T88x_L2_EXT_READ_BEATS", -+ "T88x_L2_ANY_LOOKUP", -+ "T88x_L2_READ_LOOKUP", -+ "T88x_L2_SREAD_LOOKUP", -+ "T88x_L2_READ_REPLAY", -+ "T88x_L2_READ_SNOOP", -+ "T88x_L2_READ_HIT", -+ "T88x_L2_CLEAN_MISS", -+ "T88x_L2_WRITE_LOOKUP", -+ "T88x_L2_SWRITE_LOOKUP", -+ "T88x_L2_WRITE_REPLAY", -+ "T88x_L2_WRITE_SNOOP", -+ "T88x_L2_WRITE_HIT", -+ "T88x_L2_EXT_READ_FULL", -+ "", -+ "T88x_L2_EXT_WRITE_FULL", -+ "T88x_L2_EXT_R_W_HAZARD", -+ "T88x_L2_EXT_READ", -+ "T88x_L2_EXT_READ_LINE", -+ "T88x_L2_EXT_WRITE", -+ "T88x_L2_EXT_WRITE_LINE", -+ "T88x_L2_EXT_WRITE_SMALL", -+ "T88x_L2_EXT_BARRIER", -+ "T88x_L2_EXT_AR_STALL", -+ "T88x_L2_EXT_R_BUF_FULL", -+ "T88x_L2_EXT_RD_BUF_FULL", -+ "T88x_L2_EXT_R_RAW", -+ "T88x_L2_EXT_W_STALL", -+ "T88x_L2_EXT_W_BUF_FULL", -+ "T88x_L2_EXT_R_BUF_FULL", -+ "T88x_L2_TAG_HAZARD", -+ "T88x_L2_SNOOP_FULL", -+ "T88x_L2_REPLAY_FULL" -+}; -+ -+#include "mali_kbase_gator_hwcnt_names_tmix.h" -+ -+#include "mali_kbase_gator_hwcnt_names_thex.h" -+ -+#include "mali_kbase_gator_hwcnt_names_tsix.h" -+ -+ -+#ifdef MALI_INCLUDE_TKAX -+#include "mali_kbase_gator_hwcnt_names_tkax.h" -+#endif /* MALI_INCLUDE_TKAX */ -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h -new file mode 100755 -index 000000000..bcceef4fc ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_thex.h -@@ -0,0 +1,291 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * This header was autogenerated, it should not be edited. -+ */ -+ -+#ifndef _KBASE_GATOR_HWCNT_NAMES_THEX_H_ -+#define _KBASE_GATOR_HWCNT_NAMES_THEX_H_ -+ -+static const char * const hardware_counters_mali_tHEx[] = { -+ /* Performance counters for the Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "THEx_MESSAGES_SENT", -+ "THEx_MESSAGES_RECEIVED", -+ "THEx_GPU_ACTIVE", -+ "THEx_IRQ_ACTIVE", -+ "THEx_JS0_JOBS", -+ "THEx_JS0_TASKS", -+ "THEx_JS0_ACTIVE", -+ "", -+ "THEx_JS0_WAIT_READ", -+ "THEx_JS0_WAIT_ISSUE", -+ "THEx_JS0_WAIT_DEPEND", -+ "THEx_JS0_WAIT_FINISH", -+ "THEx_JS1_JOBS", -+ "THEx_JS1_TASKS", -+ "THEx_JS1_ACTIVE", -+ "", -+ "THEx_JS1_WAIT_READ", -+ "THEx_JS1_WAIT_ISSUE", -+ "THEx_JS1_WAIT_DEPEND", -+ "THEx_JS1_WAIT_FINISH", -+ "THEx_JS2_JOBS", -+ "THEx_JS2_TASKS", -+ "THEx_JS2_ACTIVE", -+ "", -+ "THEx_JS2_WAIT_READ", -+ "THEx_JS2_WAIT_ISSUE", -+ "THEx_JS2_WAIT_DEPEND", -+ "THEx_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /* Performance counters for the Tiler */ -+ "", -+ "", -+ "", -+ "", -+ "THEx_TILER_ACTIVE", -+ "THEx_JOBS_PROCESSED", -+ "THEx_TRIANGLES", -+ "THEx_LINES", -+ "THEx_POINTS", -+ "THEx_FRONT_FACING", -+ "THEx_BACK_FACING", -+ "THEx_PRIM_VISIBLE", -+ "THEx_PRIM_CULLED", -+ "THEx_PRIM_CLIPPED", -+ "THEx_PRIM_SAT_CULLED", -+ "", -+ "", -+ "THEx_BUS_READ", -+ "", -+ "THEx_BUS_WRITE", -+ "THEx_LOADING_DESC", -+ "THEx_IDVS_POS_SHAD_REQ", -+ "THEx_IDVS_POS_SHAD_WAIT", -+ "THEx_IDVS_POS_SHAD_STALL", -+ "THEx_IDVS_POS_FIFO_FULL", -+ "THEx_PREFETCH_STALL", -+ "THEx_VCACHE_HIT", -+ "THEx_VCACHE_MISS", -+ "THEx_VCACHE_LINE_WAIT", -+ "THEx_VFETCH_POS_READ_WAIT", -+ "THEx_VFETCH_VERTEX_WAIT", -+ "THEx_VFETCH_STALL", -+ "THEx_PRIMASSY_STALL", -+ "THEx_BBOX_GEN_STALL", -+ "THEx_IDVS_VBU_HIT", -+ "THEx_IDVS_VBU_MISS", -+ "THEx_IDVS_VBU_LINE_DEALLOCATE", -+ "THEx_IDVS_VAR_SHAD_REQ", -+ "THEx_IDVS_VAR_SHAD_STALL", -+ "THEx_BINNER_STALL", -+ "THEx_ITER_STALL", -+ "THEx_COMPRESS_MISS", -+ "THEx_COMPRESS_STALL", -+ "THEx_PCACHE_HIT", -+ "THEx_PCACHE_MISS", -+ "THEx_PCACHE_MISS_STALL", -+ "THEx_PCACHE_EVICT_STALL", -+ "THEx_PMGR_PTR_WR_STALL", -+ "THEx_PMGR_PTR_RD_STALL", -+ "THEx_PMGR_CMD_WR_STALL", -+ "THEx_WRBUF_ACTIVE", -+ "THEx_WRBUF_HIT", -+ "THEx_WRBUF_MISS", -+ "THEx_WRBUF_NO_FREE_LINE_STALL", -+ "THEx_WRBUF_NO_AXI_ID_STALL", -+ "THEx_WRBUF_AXI_STALL", -+ "", -+ "", -+ "", -+ "THEx_UTLB_TRANS", -+ "THEx_UTLB_TRANS_HIT", -+ "THEx_UTLB_TRANS_STALL", -+ "THEx_UTLB_TRANS_MISS_DELAY", -+ "THEx_UTLB_MMU_REQ", -+ -+ /* Performance counters for the Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "THEx_FRAG_ACTIVE", -+ "THEx_FRAG_PRIMITIVES", -+ "THEx_FRAG_PRIM_RAST", -+ "THEx_FRAG_FPK_ACTIVE", -+ "THEx_FRAG_STARVING", -+ "THEx_FRAG_WARPS", -+ "THEx_FRAG_PARTIAL_WARPS", -+ "THEx_FRAG_QUADS_RAST", -+ "THEx_FRAG_QUADS_EZS_TEST", -+ "THEx_FRAG_QUADS_EZS_UPDATE", -+ "THEx_FRAG_QUADS_EZS_KILL", -+ "THEx_FRAG_LZS_TEST", -+ "THEx_FRAG_LZS_KILL", -+ "", -+ "THEx_FRAG_PTILES", -+ "THEx_FRAG_TRANS_ELIM", -+ "THEx_QUAD_FPK_KILLER", -+ "", -+ "THEx_COMPUTE_ACTIVE", -+ "THEx_COMPUTE_TASKS", -+ "THEx_COMPUTE_WARPS", -+ "THEx_COMPUTE_STARVING", -+ "THEx_EXEC_CORE_ACTIVE", -+ "THEx_EXEC_ACTIVE", -+ "THEx_EXEC_INSTR_COUNT", -+ "THEx_EXEC_INSTR_DIVERGED", -+ "THEx_EXEC_INSTR_STARVING", -+ "THEx_ARITH_INSTR_SINGLE_FMA", -+ "THEx_ARITH_INSTR_DOUBLE", -+ "THEx_ARITH_INSTR_MSG", -+ "THEx_ARITH_INSTR_MSG_ONLY", -+ "THEx_TEX_INSTR", -+ "THEx_TEX_INSTR_MIPMAP", -+ "THEx_TEX_INSTR_COMPRESSED", -+ "THEx_TEX_INSTR_3D", -+ "THEx_TEX_INSTR_TRILINEAR", -+ "THEx_TEX_COORD_ISSUE", -+ "THEx_TEX_COORD_STALL", -+ "THEx_TEX_STARVE_CACHE", -+ "THEx_TEX_STARVE_FILTER", -+ "THEx_LS_MEM_READ_FULL", -+ "THEx_LS_MEM_READ_SHORT", -+ "THEx_LS_MEM_WRITE_FULL", -+ "THEx_LS_MEM_WRITE_SHORT", -+ "THEx_LS_MEM_ATOMIC", -+ "THEx_VARY_INSTR", -+ "THEx_VARY_SLOT_32", -+ "THEx_VARY_SLOT_16", -+ "THEx_ATTR_INSTR", -+ "THEx_ARITH_INSTR_FP_MUL", -+ "THEx_BEATS_RD_FTC", -+ "THEx_BEATS_RD_FTC_EXT", -+ "THEx_BEATS_RD_LSC", -+ "THEx_BEATS_RD_LSC_EXT", -+ "THEx_BEATS_RD_TEX", -+ "THEx_BEATS_RD_TEX_EXT", -+ "THEx_BEATS_RD_OTHER", -+ "THEx_BEATS_WR_LSC", -+ "THEx_BEATS_WR_TIB", -+ "", -+ -+ /* Performance counters for the Memory System */ -+ "", -+ "", -+ "", -+ "", -+ "THEx_MMU_REQUESTS", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "THEx_L2_RD_MSG_IN", -+ "THEx_L2_RD_MSG_IN_STALL", -+ "THEx_L2_WR_MSG_IN", -+ "THEx_L2_WR_MSG_IN_STALL", -+ "THEx_L2_SNP_MSG_IN", -+ "THEx_L2_SNP_MSG_IN_STALL", -+ "THEx_L2_RD_MSG_OUT", -+ "THEx_L2_RD_MSG_OUT_STALL", -+ "THEx_L2_WR_MSG_OUT", -+ "THEx_L2_ANY_LOOKUP", -+ "THEx_L2_READ_LOOKUP", -+ "THEx_L2_WRITE_LOOKUP", -+ "THEx_L2_EXT_SNOOP_LOOKUP", -+ "THEx_L2_EXT_READ", -+ "THEx_L2_EXT_READ_NOSNP", -+ "THEx_L2_EXT_READ_UNIQUE", -+ "THEx_L2_EXT_READ_BEATS", -+ "THEx_L2_EXT_AR_STALL", -+ "THEx_L2_EXT_AR_CNT_Q1", -+ "THEx_L2_EXT_AR_CNT_Q2", -+ "THEx_L2_EXT_AR_CNT_Q3", -+ "THEx_L2_EXT_RRESP_0_127", -+ "THEx_L2_EXT_RRESP_128_191", -+ "THEx_L2_EXT_RRESP_192_255", -+ "THEx_L2_EXT_RRESP_256_319", -+ "THEx_L2_EXT_RRESP_320_383", -+ "THEx_L2_EXT_WRITE", -+ "THEx_L2_EXT_WRITE_NOSNP_FULL", -+ "THEx_L2_EXT_WRITE_NOSNP_PTL", -+ "THEx_L2_EXT_WRITE_SNP_FULL", -+ "THEx_L2_EXT_WRITE_SNP_PTL", -+ "THEx_L2_EXT_WRITE_BEATS", -+ "THEx_L2_EXT_W_STALL", -+ "THEx_L2_EXT_AW_CNT_Q1", -+ "THEx_L2_EXT_AW_CNT_Q2", -+ "THEx_L2_EXT_AW_CNT_Q3", -+ "THEx_L2_EXT_SNOOP", -+ "THEx_L2_EXT_SNOOP_STALL", -+ "THEx_L2_EXT_SNOOP_RESP_CLEAN", -+ "THEx_L2_EXT_SNOOP_RESP_DATA", -+ "THEx_L2_EXT_SNOOP_INTERNAL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+}; -+ -+#endif /* _KBASE_GATOR_HWCNT_NAMES_THEX_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h -new file mode 100755 -index 000000000..5ea06770f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tmix.h -@@ -0,0 +1,291 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * This header was autogenerated, it should not be edited. -+ */ -+ -+#ifndef _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ -+#define _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ -+ -+static const char * const hardware_counters_mali_tMIx[] = { -+ /* Performance counters for the Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "TMIx_MESSAGES_SENT", -+ "TMIx_MESSAGES_RECEIVED", -+ "TMIx_GPU_ACTIVE", -+ "TMIx_IRQ_ACTIVE", -+ "TMIx_JS0_JOBS", -+ "TMIx_JS0_TASKS", -+ "TMIx_JS0_ACTIVE", -+ "", -+ "TMIx_JS0_WAIT_READ", -+ "TMIx_JS0_WAIT_ISSUE", -+ "TMIx_JS0_WAIT_DEPEND", -+ "TMIx_JS0_WAIT_FINISH", -+ "TMIx_JS1_JOBS", -+ "TMIx_JS1_TASKS", -+ "TMIx_JS1_ACTIVE", -+ "", -+ "TMIx_JS1_WAIT_READ", -+ "TMIx_JS1_WAIT_ISSUE", -+ "TMIx_JS1_WAIT_DEPEND", -+ "TMIx_JS1_WAIT_FINISH", -+ "TMIx_JS2_JOBS", -+ "TMIx_JS2_TASKS", -+ "TMIx_JS2_ACTIVE", -+ "", -+ "TMIx_JS2_WAIT_READ", -+ "TMIx_JS2_WAIT_ISSUE", -+ "TMIx_JS2_WAIT_DEPEND", -+ "TMIx_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /* Performance counters for the Tiler */ -+ "", -+ "", -+ "", -+ "", -+ "TMIx_TILER_ACTIVE", -+ "TMIx_JOBS_PROCESSED", -+ "TMIx_TRIANGLES", -+ "TMIx_LINES", -+ "TMIx_POINTS", -+ "TMIx_FRONT_FACING", -+ "TMIx_BACK_FACING", -+ "TMIx_PRIM_VISIBLE", -+ "TMIx_PRIM_CULLED", -+ "TMIx_PRIM_CLIPPED", -+ "TMIx_PRIM_SAT_CULLED", -+ "", -+ "", -+ "TMIx_BUS_READ", -+ "", -+ "TMIx_BUS_WRITE", -+ "TMIx_LOADING_DESC", -+ "TMIx_IDVS_POS_SHAD_REQ", -+ "TMIx_IDVS_POS_SHAD_WAIT", -+ "TMIx_IDVS_POS_SHAD_STALL", -+ "TMIx_IDVS_POS_FIFO_FULL", -+ "TMIx_PREFETCH_STALL", -+ "TMIx_VCACHE_HIT", -+ "TMIx_VCACHE_MISS", -+ "TMIx_VCACHE_LINE_WAIT", -+ "TMIx_VFETCH_POS_READ_WAIT", -+ "TMIx_VFETCH_VERTEX_WAIT", -+ "TMIx_VFETCH_STALL", -+ "TMIx_PRIMASSY_STALL", -+ "TMIx_BBOX_GEN_STALL", -+ "TMIx_IDVS_VBU_HIT", -+ "TMIx_IDVS_VBU_MISS", -+ "TMIx_IDVS_VBU_LINE_DEALLOCATE", -+ "TMIx_IDVS_VAR_SHAD_REQ", -+ "TMIx_IDVS_VAR_SHAD_STALL", -+ "TMIx_BINNER_STALL", -+ "TMIx_ITER_STALL", -+ "TMIx_COMPRESS_MISS", -+ "TMIx_COMPRESS_STALL", -+ "TMIx_PCACHE_HIT", -+ "TMIx_PCACHE_MISS", -+ "TMIx_PCACHE_MISS_STALL", -+ "TMIx_PCACHE_EVICT_STALL", -+ "TMIx_PMGR_PTR_WR_STALL", -+ "TMIx_PMGR_PTR_RD_STALL", -+ "TMIx_PMGR_CMD_WR_STALL", -+ "TMIx_WRBUF_ACTIVE", -+ "TMIx_WRBUF_HIT", -+ "TMIx_WRBUF_MISS", -+ "TMIx_WRBUF_NO_FREE_LINE_STALL", -+ "TMIx_WRBUF_NO_AXI_ID_STALL", -+ "TMIx_WRBUF_AXI_STALL", -+ "", -+ "", -+ "", -+ "TMIx_UTLB_TRANS", -+ "TMIx_UTLB_TRANS_HIT", -+ "TMIx_UTLB_TRANS_STALL", -+ "TMIx_UTLB_TRANS_MISS_DELAY", -+ "TMIx_UTLB_MMU_REQ", -+ -+ /* Performance counters for the Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "TMIx_FRAG_ACTIVE", -+ "TMIx_FRAG_PRIMITIVES", -+ "TMIx_FRAG_PRIM_RAST", -+ "TMIx_FRAG_FPK_ACTIVE", -+ "TMIx_FRAG_STARVING", -+ "TMIx_FRAG_WARPS", -+ "TMIx_FRAG_PARTIAL_WARPS", -+ "TMIx_FRAG_QUADS_RAST", -+ "TMIx_FRAG_QUADS_EZS_TEST", -+ "TMIx_FRAG_QUADS_EZS_UPDATE", -+ "TMIx_FRAG_QUADS_EZS_KILL", -+ "TMIx_FRAG_LZS_TEST", -+ "TMIx_FRAG_LZS_KILL", -+ "", -+ "TMIx_FRAG_PTILES", -+ "TMIx_FRAG_TRANS_ELIM", -+ "TMIx_QUAD_FPK_KILLER", -+ "", -+ "TMIx_COMPUTE_ACTIVE", -+ "TMIx_COMPUTE_TASKS", -+ "TMIx_COMPUTE_WARPS", -+ "TMIx_COMPUTE_STARVING", -+ "TMIx_EXEC_CORE_ACTIVE", -+ "TMIx_EXEC_ACTIVE", -+ "TMIx_EXEC_INSTR_COUNT", -+ "TMIx_EXEC_INSTR_DIVERGED", -+ "TMIx_EXEC_INSTR_STARVING", -+ "TMIx_ARITH_INSTR_SINGLE_FMA", -+ "TMIx_ARITH_INSTR_DOUBLE", -+ "TMIx_ARITH_INSTR_MSG", -+ "TMIx_ARITH_INSTR_MSG_ONLY", -+ "TMIx_TEX_INSTR", -+ "TMIx_TEX_INSTR_MIPMAP", -+ "TMIx_TEX_INSTR_COMPRESSED", -+ "TMIx_TEX_INSTR_3D", -+ "TMIx_TEX_INSTR_TRILINEAR", -+ "TMIx_TEX_COORD_ISSUE", -+ "TMIx_TEX_COORD_STALL", -+ "TMIx_TEX_STARVE_CACHE", -+ "TMIx_TEX_STARVE_FILTER", -+ "TMIx_LS_MEM_READ_FULL", -+ "TMIx_LS_MEM_READ_SHORT", -+ "TMIx_LS_MEM_WRITE_FULL", -+ "TMIx_LS_MEM_WRITE_SHORT", -+ "TMIx_LS_MEM_ATOMIC", -+ "TMIx_VARY_INSTR", -+ "TMIx_VARY_SLOT_32", -+ "TMIx_VARY_SLOT_16", -+ "TMIx_ATTR_INSTR", -+ "TMIx_ARITH_INSTR_FP_MUL", -+ "TMIx_BEATS_RD_FTC", -+ "TMIx_BEATS_RD_FTC_EXT", -+ "TMIx_BEATS_RD_LSC", -+ "TMIx_BEATS_RD_LSC_EXT", -+ "TMIx_BEATS_RD_TEX", -+ "TMIx_BEATS_RD_TEX_EXT", -+ "TMIx_BEATS_RD_OTHER", -+ "TMIx_BEATS_WR_LSC", -+ "TMIx_BEATS_WR_TIB", -+ "", -+ -+ /* Performance counters for the Memory System */ -+ "", -+ "", -+ "", -+ "", -+ "TMIx_MMU_REQUESTS", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "TMIx_L2_RD_MSG_IN", -+ "TMIx_L2_RD_MSG_IN_STALL", -+ "TMIx_L2_WR_MSG_IN", -+ "TMIx_L2_WR_MSG_IN_STALL", -+ "TMIx_L2_SNP_MSG_IN", -+ "TMIx_L2_SNP_MSG_IN_STALL", -+ "TMIx_L2_RD_MSG_OUT", -+ "TMIx_L2_RD_MSG_OUT_STALL", -+ "TMIx_L2_WR_MSG_OUT", -+ "TMIx_L2_ANY_LOOKUP", -+ "TMIx_L2_READ_LOOKUP", -+ "TMIx_L2_WRITE_LOOKUP", -+ "TMIx_L2_EXT_SNOOP_LOOKUP", -+ "TMIx_L2_EXT_READ", -+ "TMIx_L2_EXT_READ_NOSNP", -+ "TMIx_L2_EXT_READ_UNIQUE", -+ "TMIx_L2_EXT_READ_BEATS", -+ "TMIx_L2_EXT_AR_STALL", -+ "TMIx_L2_EXT_AR_CNT_Q1", -+ "TMIx_L2_EXT_AR_CNT_Q2", -+ "TMIx_L2_EXT_AR_CNT_Q3", -+ "TMIx_L2_EXT_RRESP_0_127", -+ "TMIx_L2_EXT_RRESP_128_191", -+ "TMIx_L2_EXT_RRESP_192_255", -+ "TMIx_L2_EXT_RRESP_256_319", -+ "TMIx_L2_EXT_RRESP_320_383", -+ "TMIx_L2_EXT_WRITE", -+ "TMIx_L2_EXT_WRITE_NOSNP_FULL", -+ "TMIx_L2_EXT_WRITE_NOSNP_PTL", -+ "TMIx_L2_EXT_WRITE_SNP_FULL", -+ "TMIx_L2_EXT_WRITE_SNP_PTL", -+ "TMIx_L2_EXT_WRITE_BEATS", -+ "TMIx_L2_EXT_W_STALL", -+ "TMIx_L2_EXT_AW_CNT_Q1", -+ "TMIx_L2_EXT_AW_CNT_Q2", -+ "TMIx_L2_EXT_AW_CNT_Q3", -+ "TMIx_L2_EXT_SNOOP", -+ "TMIx_L2_EXT_SNOOP_STALL", -+ "TMIx_L2_EXT_SNOOP_RESP_CLEAN", -+ "TMIx_L2_EXT_SNOOP_RESP_DATA", -+ "TMIx_L2_EXT_SNOOP_INTERNAL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+}; -+ -+#endif /* _KBASE_GATOR_HWCNT_NAMES_TMIX_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h -new file mode 100755 -index 000000000..be09c4556 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gator_hwcnt_names_tsix.h -@@ -0,0 +1,291 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * This header was autogenerated, it should not be edited. -+ */ -+ -+#ifndef _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ -+#define _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ -+ -+static const char * const hardware_counters_mali_tSIx[] = { -+ /* Performance counters for the Job Manager */ -+ "", -+ "", -+ "", -+ "", -+ "TSIx_MESSAGES_SENT", -+ "TSIx_MESSAGES_RECEIVED", -+ "TSIx_GPU_ACTIVE", -+ "TSIx_IRQ_ACTIVE", -+ "TSIx_JS0_JOBS", -+ "TSIx_JS0_TASKS", -+ "TSIx_JS0_ACTIVE", -+ "", -+ "TSIx_JS0_WAIT_READ", -+ "TSIx_JS0_WAIT_ISSUE", -+ "TSIx_JS0_WAIT_DEPEND", -+ "TSIx_JS0_WAIT_FINISH", -+ "TSIx_JS1_JOBS", -+ "TSIx_JS1_TASKS", -+ "TSIx_JS1_ACTIVE", -+ "", -+ "TSIx_JS1_WAIT_READ", -+ "TSIx_JS1_WAIT_ISSUE", -+ "TSIx_JS1_WAIT_DEPEND", -+ "TSIx_JS1_WAIT_FINISH", -+ "TSIx_JS2_JOBS", -+ "TSIx_JS2_TASKS", -+ "TSIx_JS2_ACTIVE", -+ "", -+ "TSIx_JS2_WAIT_READ", -+ "TSIx_JS2_WAIT_ISSUE", -+ "TSIx_JS2_WAIT_DEPEND", -+ "TSIx_JS2_WAIT_FINISH", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ -+ /* Performance counters for the Tiler */ -+ "", -+ "", -+ "", -+ "", -+ "TSIx_TILER_ACTIVE", -+ "TSIx_JOBS_PROCESSED", -+ "TSIx_TRIANGLES", -+ "TSIx_LINES", -+ "TSIx_POINTS", -+ "TSIx_FRONT_FACING", -+ "TSIx_BACK_FACING", -+ "TSIx_PRIM_VISIBLE", -+ "TSIx_PRIM_CULLED", -+ "TSIx_PRIM_CLIPPED", -+ "TSIx_PRIM_SAT_CULLED", -+ "", -+ "", -+ "TSIx_BUS_READ", -+ "", -+ "TSIx_BUS_WRITE", -+ "TSIx_LOADING_DESC", -+ "TSIx_IDVS_POS_SHAD_REQ", -+ "TSIx_IDVS_POS_SHAD_WAIT", -+ "TSIx_IDVS_POS_SHAD_STALL", -+ "TSIx_IDVS_POS_FIFO_FULL", -+ "TSIx_PREFETCH_STALL", -+ "TSIx_VCACHE_HIT", -+ "TSIx_VCACHE_MISS", -+ "TSIx_VCACHE_LINE_WAIT", -+ "TSIx_VFETCH_POS_READ_WAIT", -+ "TSIx_VFETCH_VERTEX_WAIT", -+ "TSIx_VFETCH_STALL", -+ "TSIx_PRIMASSY_STALL", -+ "TSIx_BBOX_GEN_STALL", -+ "TSIx_IDVS_VBU_HIT", -+ "TSIx_IDVS_VBU_MISS", -+ "TSIx_IDVS_VBU_LINE_DEALLOCATE", -+ "TSIx_IDVS_VAR_SHAD_REQ", -+ "TSIx_IDVS_VAR_SHAD_STALL", -+ "TSIx_BINNER_STALL", -+ "TSIx_ITER_STALL", -+ "TSIx_COMPRESS_MISS", -+ "TSIx_COMPRESS_STALL", -+ "TSIx_PCACHE_HIT", -+ "TSIx_PCACHE_MISS", -+ "TSIx_PCACHE_MISS_STALL", -+ "TSIx_PCACHE_EVICT_STALL", -+ "TSIx_PMGR_PTR_WR_STALL", -+ "TSIx_PMGR_PTR_RD_STALL", -+ "TSIx_PMGR_CMD_WR_STALL", -+ "TSIx_WRBUF_ACTIVE", -+ "TSIx_WRBUF_HIT", -+ "TSIx_WRBUF_MISS", -+ "TSIx_WRBUF_NO_FREE_LINE_STALL", -+ "TSIx_WRBUF_NO_AXI_ID_STALL", -+ "TSIx_WRBUF_AXI_STALL", -+ "", -+ "", -+ "", -+ "TSIx_UTLB_TRANS", -+ "TSIx_UTLB_TRANS_HIT", -+ "TSIx_UTLB_TRANS_STALL", -+ "TSIx_UTLB_TRANS_MISS_DELAY", -+ "TSIx_UTLB_MMU_REQ", -+ -+ /* Performance counters for the Shader Core */ -+ "", -+ "", -+ "", -+ "", -+ "TSIx_FRAG_ACTIVE", -+ "TSIx_FRAG_PRIMITIVES", -+ "TSIx_FRAG_PRIM_RAST", -+ "TSIx_FRAG_FPK_ACTIVE", -+ "TSIx_FRAG_STARVING", -+ "TSIx_FRAG_WARPS", -+ "TSIx_FRAG_PARTIAL_WARPS", -+ "TSIx_FRAG_QUADS_RAST", -+ "TSIx_FRAG_QUADS_EZS_TEST", -+ "TSIx_FRAG_QUADS_EZS_UPDATE", -+ "TSIx_FRAG_QUADS_EZS_KILL", -+ "TSIx_FRAG_LZS_TEST", -+ "TSIx_FRAG_LZS_KILL", -+ "", -+ "TSIx_FRAG_PTILES", -+ "TSIx_FRAG_TRANS_ELIM", -+ "TSIx_QUAD_FPK_KILLER", -+ "", -+ "TSIx_COMPUTE_ACTIVE", -+ "TSIx_COMPUTE_TASKS", -+ "TSIx_COMPUTE_WARPS", -+ "TSIx_COMPUTE_STARVING", -+ "TSIx_EXEC_CORE_ACTIVE", -+ "TSIx_EXEC_ACTIVE", -+ "TSIx_EXEC_INSTR_COUNT", -+ "TSIx_EXEC_INSTR_DIVERGED", -+ "TSIx_EXEC_INSTR_STARVING", -+ "TSIx_ARITH_INSTR_SINGLE_FMA", -+ "TSIx_ARITH_INSTR_DOUBLE", -+ "TSIx_ARITH_INSTR_MSG", -+ "TSIx_ARITH_INSTR_MSG_ONLY", -+ "TSIx_TEX_MSGI_NUM_QUADS", -+ "TSIx_TEX_DFCH_NUM_PASSES", -+ "TSIx_TEX_DFCH_NUM_PASSES_MISS", -+ "TSIx_TEX_DFCH_NUM_PASSES_MIP_MAP", -+ "TSIx_TEX_TIDX_NUM_SPLIT_MIP_MAP", -+ "TSIx_TEX_TFCH_NUM_LINES_FETCHED", -+ "TSIx_TEX_TFCH_NUM_LINES_FETCHED_BLOCK_COMPRESSED", -+ "TSIx_TEX_TFCH_NUM_OPERATIONS", -+ "TSIx_TEX_FILT_NUM_OPERATIONS", -+ "TSIx_LS_MEM_READ_FULL", -+ "TSIx_LS_MEM_READ_SHORT", -+ "TSIx_LS_MEM_WRITE_FULL", -+ "TSIx_LS_MEM_WRITE_SHORT", -+ "TSIx_LS_MEM_ATOMIC", -+ "TSIx_VARY_INSTR", -+ "TSIx_VARY_SLOT_32", -+ "TSIx_VARY_SLOT_16", -+ "TSIx_ATTR_INSTR", -+ "TSIx_ARITH_INSTR_FP_MUL", -+ "TSIx_BEATS_RD_FTC", -+ "TSIx_BEATS_RD_FTC_EXT", -+ "TSIx_BEATS_RD_LSC", -+ "TSIx_BEATS_RD_LSC_EXT", -+ "TSIx_BEATS_RD_TEX", -+ "TSIx_BEATS_RD_TEX_EXT", -+ "TSIx_BEATS_RD_OTHER", -+ "TSIx_BEATS_WR_LSC", -+ "TSIx_BEATS_WR_TIB", -+ "", -+ -+ /* Performance counters for the Memory System */ -+ "", -+ "", -+ "", -+ "", -+ "TSIx_MMU_REQUESTS", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "TSIx_L2_RD_MSG_IN", -+ "TSIx_L2_RD_MSG_IN_STALL", -+ "TSIx_L2_WR_MSG_IN", -+ "TSIx_L2_WR_MSG_IN_STALL", -+ "TSIx_L2_SNP_MSG_IN", -+ "TSIx_L2_SNP_MSG_IN_STALL", -+ "TSIx_L2_RD_MSG_OUT", -+ "TSIx_L2_RD_MSG_OUT_STALL", -+ "TSIx_L2_WR_MSG_OUT", -+ "TSIx_L2_ANY_LOOKUP", -+ "TSIx_L2_READ_LOOKUP", -+ "TSIx_L2_WRITE_LOOKUP", -+ "TSIx_L2_EXT_SNOOP_LOOKUP", -+ "TSIx_L2_EXT_READ", -+ "TSIx_L2_EXT_READ_NOSNP", -+ "TSIx_L2_EXT_READ_UNIQUE", -+ "TSIx_L2_EXT_READ_BEATS", -+ "TSIx_L2_EXT_AR_STALL", -+ "TSIx_L2_EXT_AR_CNT_Q1", -+ "TSIx_L2_EXT_AR_CNT_Q2", -+ "TSIx_L2_EXT_AR_CNT_Q3", -+ "TSIx_L2_EXT_RRESP_0_127", -+ "TSIx_L2_EXT_RRESP_128_191", -+ "TSIx_L2_EXT_RRESP_192_255", -+ "TSIx_L2_EXT_RRESP_256_319", -+ "TSIx_L2_EXT_RRESP_320_383", -+ "TSIx_L2_EXT_WRITE", -+ "TSIx_L2_EXT_WRITE_NOSNP_FULL", -+ "TSIx_L2_EXT_WRITE_NOSNP_PTL", -+ "TSIx_L2_EXT_WRITE_SNP_FULL", -+ "TSIx_L2_EXT_WRITE_SNP_PTL", -+ "TSIx_L2_EXT_WRITE_BEATS", -+ "TSIx_L2_EXT_W_STALL", -+ "TSIx_L2_EXT_AW_CNT_Q1", -+ "TSIx_L2_EXT_AW_CNT_Q2", -+ "TSIx_L2_EXT_AW_CNT_Q3", -+ "TSIx_L2_EXT_SNOOP", -+ "TSIx_L2_EXT_SNOOP_STALL", -+ "TSIx_L2_EXT_SNOOP_RESP_CLEAN", -+ "TSIx_L2_EXT_SNOOP_RESP_DATA", -+ "TSIx_L2_EXT_SNOOP_INTERNAL", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+ "", -+}; -+ -+#endif /* _KBASE_GATOR_HWCNT_NAMES_TSIX_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h b/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h -new file mode 100755 -index 000000000..42f0111c4 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_id.h -@@ -0,0 +1,123 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+#ifndef _KBASE_GPU_ID_H_ -+#define _KBASE_GPU_ID_H_ -+ -+/* GPU_ID register */ -+#define GPU_ID_VERSION_STATUS_SHIFT 0 -+#define GPU_ID_VERSION_MINOR_SHIFT 4 -+#define GPU_ID_VERSION_MAJOR_SHIFT 12 -+#define GPU_ID_VERSION_PRODUCT_ID_SHIFT 16 -+#define GPU_ID_VERSION_STATUS (0xF << GPU_ID_VERSION_STATUS_SHIFT) -+#define GPU_ID_VERSION_MINOR (0xFF << GPU_ID_VERSION_MINOR_SHIFT) -+#define GPU_ID_VERSION_MAJOR (0xF << GPU_ID_VERSION_MAJOR_SHIFT) -+#define GPU_ID_VERSION_PRODUCT_ID (0xFFFF << GPU_ID_VERSION_PRODUCT_ID_SHIFT) -+ -+/* Values for GPU_ID_VERSION_PRODUCT_ID bitfield */ -+#define GPU_ID_PI_T60X 0x6956 -+#define GPU_ID_PI_T62X 0x0620 -+#define GPU_ID_PI_T76X 0x0750 -+#define GPU_ID_PI_T72X 0x0720 -+#define GPU_ID_PI_TFRX 0x0880 -+#define GPU_ID_PI_T86X 0x0860 -+#define GPU_ID_PI_T82X 0x0820 -+#define GPU_ID_PI_T83X 0x0830 -+ -+/* New GPU ID format when PRODUCT_ID is >= 0x1000 (and not 0x6956) */ -+#define GPU_ID_PI_NEW_FORMAT_START 0x1000 -+#define GPU_ID_IS_NEW_FORMAT(product_id) ((product_id) != GPU_ID_PI_T60X && \ -+ (product_id) >= \ -+ GPU_ID_PI_NEW_FORMAT_START) -+ -+#define GPU_ID2_VERSION_STATUS_SHIFT 0 -+#define GPU_ID2_VERSION_MINOR_SHIFT 4 -+#define GPU_ID2_VERSION_MAJOR_SHIFT 12 -+#define GPU_ID2_PRODUCT_MAJOR_SHIFT 16 -+#define GPU_ID2_ARCH_REV_SHIFT 20 -+#define GPU_ID2_ARCH_MINOR_SHIFT 24 -+#define GPU_ID2_ARCH_MAJOR_SHIFT 28 -+#define GPU_ID2_VERSION_STATUS (0xF << GPU_ID2_VERSION_STATUS_SHIFT) -+#define GPU_ID2_VERSION_MINOR (0xFF << GPU_ID2_VERSION_MINOR_SHIFT) -+#define GPU_ID2_VERSION_MAJOR (0xF << GPU_ID2_VERSION_MAJOR_SHIFT) -+#define GPU_ID2_PRODUCT_MAJOR (0xF << GPU_ID2_PRODUCT_MAJOR_SHIFT) -+#define GPU_ID2_ARCH_REV (0xF << GPU_ID2_ARCH_REV_SHIFT) -+#define GPU_ID2_ARCH_MINOR (0xF << GPU_ID2_ARCH_MINOR_SHIFT) -+#define GPU_ID2_ARCH_MAJOR (0xF << GPU_ID2_ARCH_MAJOR_SHIFT) -+#define GPU_ID2_PRODUCT_MODEL (GPU_ID2_ARCH_MAJOR | GPU_ID2_PRODUCT_MAJOR) -+#define GPU_ID2_VERSION (GPU_ID2_VERSION_MAJOR | \ -+ GPU_ID2_VERSION_MINOR | \ -+ GPU_ID2_VERSION_STATUS) -+ -+/* Helper macro to create a partial GPU_ID (new format) that defines -+ a product ignoring its version. */ -+#define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ -+ (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ -+ ((arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ -+ ((arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ -+ ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) -+ -+/* Helper macro to create a partial GPU_ID (new format) that specifies the -+ revision (major, minor, status) of a product */ -+#define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ -+ (((version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ -+ ((version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ -+ ((version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) -+ -+/* Helper macro to create a complete GPU_ID (new format) */ -+#define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ -+ version_major, version_minor, version_status) \ -+ (GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, \ -+ product_major) | \ -+ GPU_ID2_VERSION_MAKE(version_major, version_minor, \ -+ version_status)) -+ -+/* Helper macro to create a partial GPU_ID (new format) that identifies -+ a particular GPU model by its arch_major and product_major. */ -+#define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ -+ (((arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ -+ ((product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) -+ -+/* Strip off the non-relevant bits from a product_id value and make it suitable -+ for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU -+ model. */ -+#define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ -+ (((product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ -+ GPU_ID2_PRODUCT_MODEL) -+ -+#define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6u, 0) -+#define GPU_ID2_PRODUCT_THEX GPU_ID2_MODEL_MAKE(6u, 1) -+#define GPU_ID2_PRODUCT_TSIX GPU_ID2_MODEL_MAKE(7u, 0) -+#ifdef MALI_INCLUDE_TKAX -+#define GPU_ID2_PRODUCT_TKAX GPU_ID2_MODEL_MAKE(9u, 0) -+#endif /* MALI_INCLUDE_TKAX */ -+#ifdef MALI_INCLUDE_TTRX -+#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(10u, 0) -+#endif /* MALI_INCLUDE_TTRX */ -+ -+/* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ -+#define GPU_ID_S_15DEV0 0x1 -+#define GPU_ID_S_EAC 0x2 -+ -+/* Helper macro to create a GPU_ID assuming valid values for id, major, -+ minor, status */ -+#define GPU_ID_MAKE(id, major, minor, status) \ -+ (((id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ -+ ((major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ -+ ((minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ -+ ((status) << GPU_ID_VERSION_STATUS_SHIFT)) -+ -+#endif /* _KBASE_GPU_ID_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c -new file mode 100755 -index 000000000..6df0a1cb1 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c -@@ -0,0 +1,97 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+ -+#ifdef CONFIG_DEBUG_FS -+/** Show callback for the @c gpu_memory debugfs file. -+ * -+ * This function is called to get the contents of the @c gpu_memory debugfs -+ * file. This is a report of current gpu memory usage. -+ * -+ * @param sfile The debugfs entry -+ * @param data Data associated with the entry -+ * -+ * @return 0 if successfully prints data in debugfs entry file -+ * -1 if it encountered an error -+ */ -+ -+static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) -+{ -+ struct list_head *entry; -+ const struct list_head *kbdev_list; -+ -+ kbdev_list = kbase_dev_list_get(); -+ list_for_each(entry, kbdev_list) { -+ struct kbase_device *kbdev = NULL; -+ struct kbasep_kctx_list_element *element; -+ -+ kbdev = list_entry(entry, struct kbase_device, entry); -+ /* output the total memory usage and cap for this device */ -+ seq_printf(sfile, "%-16s %10u\n", -+ kbdev->devname, -+ atomic_read(&(kbdev->memdev.used_pages))); -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_for_each_entry(element, &kbdev->kctx_list, link) { -+ /* output the memory usage and cap for each kctx -+ * opened on this device */ -+ seq_printf(sfile, " %s-0x%p %10u\n", -+ "kctx", -+ element->kctx, -+ atomic_read(&(element->kctx->used_pages))); -+ } -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } -+ kbase_dev_list_put(kbdev_list); -+ return 0; -+} -+ -+/* -+ * File operations related to debugfs entry for gpu_memory -+ */ -+static int kbasep_gpu_memory_debugfs_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbasep_gpu_memory_seq_show , NULL); -+} -+ -+static const struct file_operations kbasep_gpu_memory_debugfs_fops = { -+ .open = kbasep_gpu_memory_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+/* -+ * Initialize debugfs entry for gpu_memory -+ */ -+void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("gpu_memory", S_IRUGO, -+ kbdev->mali_debugfs_directory, NULL, -+ &kbasep_gpu_memory_debugfs_fops); -+ return; -+} -+ -+#else -+/* -+ * Stub functions for when debugfs is disabled -+ */ -+void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev) -+{ -+ return; -+} -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h -new file mode 100755 -index 000000000..7045693eb ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h -@@ -0,0 +1,37 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2014, 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_gpu_memory_debugfs.h -+ * Header file for gpu_memory entry in debugfs -+ * -+ */ -+ -+#ifndef _KBASE_GPU_MEMORY_DEBUGFS_H -+#define _KBASE_GPU_MEMORY_DEBUGFS_H -+ -+#include -+#include -+ -+/** -+ * @brief Initialize gpu_memory debugfs entry -+ */ -+void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); -+ -+#endif /*_KBASE_GPU_MEMORY_DEBUGFS_H*/ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c -new file mode 100755 -index 000000000..a947a2e03 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c -@@ -0,0 +1,510 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Base kernel property query APIs -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include "mali_kbase_ioctl.h" -+#include -+ -+/** -+ * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. -+ * @value: The value from which to extract bits. -+ * @offset: The first bit to extract (0 being the LSB). -+ * @size: The number of bits to extract. -+ * -+ * Context: @offset + @size <= 32. -+ * -+ * Return: Bits [@offset, @offset + @size) from @value. -+ */ -+/* from mali_cdsb.h */ -+#define KBASE_UBFX32(value, offset, size) \ -+ (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) -+ -+int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props) -+{ -+ kbase_gpu_clk_speed_func get_gpu_speed_mhz; -+ u32 gpu_speed_mhz; -+ int rc = 1; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL != kbase_props); -+ -+ /* Current GPU speed is requested from the system integrator via the GPU_SPEED_FUNC function. -+ * If that function fails, or the function is not provided by the system integrator, we report the maximum -+ * GPU speed as specified by GPU_FREQ_KHZ_MAX. -+ */ -+ get_gpu_speed_mhz = (kbase_gpu_clk_speed_func) GPU_SPEED_FUNC; -+ if (get_gpu_speed_mhz != NULL) { -+ rc = get_gpu_speed_mhz(&gpu_speed_mhz); -+#ifdef CONFIG_MALI_DEBUG -+ /* Issue a warning message when the reported GPU speed falls outside the min/max range */ -+ if (rc == 0) { -+ u32 gpu_speed_khz = gpu_speed_mhz * 1000; -+ -+ if (gpu_speed_khz < kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min || -+ gpu_speed_khz > kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max) -+ dev_warn(kctx->kbdev->dev, "GPU Speed is outside of min/max range (got %lu Khz, min %lu Khz, max %lu Khz)\n", -+ (unsigned long)gpu_speed_khz, -+ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_min, -+ (unsigned long)kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max); -+ } -+#endif /* CONFIG_MALI_DEBUG */ -+ } -+ if (kctx->kbdev->clock) { -+ gpu_speed_mhz = clk_get_rate(kctx->kbdev->clock) / 1000000; -+ rc = 0; -+ } -+ if (rc != 0) -+ gpu_speed_mhz = kctx->kbdev->gpu_props.props.core_props.gpu_freq_khz_max / 1000; -+ -+ kctx->kbdev->gpu_props.props.core_props.gpu_speed_mhz = gpu_speed_mhz; -+ -+ memcpy(&kbase_props->props, &kctx->kbdev->gpu_props.props, sizeof(kbase_props->props)); -+ -+ /* Before API 8.2 they expect L3 cache info here, which was always 0 */ -+ if (kctx->api_version < KBASE_API_VERSION(8, 2)) -+ kbase_props->props.raw_props.suspend_size = 0; -+ -+ return 0; -+} -+ -+static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props) -+{ -+ struct mali_base_gpu_coherent_group *current_group; -+ u64 group_present; -+ u64 group_mask; -+ u64 first_set, first_set_prev; -+ u32 num_groups = 0; -+ -+ KBASE_DEBUG_ASSERT(NULL != props); -+ -+ props->coherency_info.coherency = props->raw_props.mem_features; -+ props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); -+ -+ if (props->coherency_info.coherency & GROUPS_L2_COHERENT) { -+ /* Group is l2 coherent */ -+ group_present = props->raw_props.l2_present; -+ } else { -+ /* Group is l1 coherent */ -+ group_present = props->raw_props.shader_present; -+ } -+ -+ /* -+ * The coherent group mask can be computed from the l2 present -+ * register. -+ * -+ * For the coherent group n: -+ * group_mask[n] = (first_set[n] - 1) & ~(first_set[n-1] - 1) -+ * where first_set is group_present with only its nth set-bit kept -+ * (i.e. the position from where a new group starts). -+ * -+ * For instance if the groups are l2 coherent and l2_present=0x0..01111: -+ * The first mask is: -+ * group_mask[1] = (first_set[1] - 1) & ~(first_set[0] - 1) -+ * = (0x0..010 - 1) & ~(0x0..01 - 1) -+ * = 0x0..00f -+ * The second mask is: -+ * group_mask[2] = (first_set[2] - 1) & ~(first_set[1] - 1) -+ * = (0x0..100 - 1) & ~(0x0..010 - 1) -+ * = 0x0..0f0 -+ * And so on until all the bits from group_present have been cleared -+ * (i.e. there is no group left). -+ */ -+ -+ current_group = props->coherency_info.group; -+ first_set = group_present & ~(group_present - 1); -+ -+ while (group_present != 0 && num_groups < BASE_MAX_COHERENT_GROUPS) { -+ group_present -= first_set; /* Clear the current group bit */ -+ first_set_prev = first_set; -+ -+ first_set = group_present & ~(group_present - 1); -+ group_mask = (first_set - 1) & ~(first_set_prev - 1); -+ -+ /* Populate the coherent_group structure for each group */ -+ current_group->core_mask = group_mask & props->raw_props.shader_present; -+ current_group->num_cores = hweight64(current_group->core_mask); -+ -+ num_groups++; -+ current_group++; -+ } -+ -+ if (group_present != 0) -+ pr_warn("Too many coherent groups (keeping only %d groups).\n", BASE_MAX_COHERENT_GROUPS); -+ -+ props->coherency_info.num_groups = num_groups; -+} -+ -+/** -+ * kbase_gpuprops_get_props - Get the GPU configuration -+ * @gpu_props: The &base_gpu_props structure -+ * @kbdev: The &struct kbase_device structure for the device -+ * -+ * Fill the &base_gpu_props structure with values from the GPU configuration -+ * registers. Only the raw properties are filled in this function -+ */ -+static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) -+{ -+ struct kbase_gpuprops_regdump regdump; -+ int i; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(NULL != gpu_props); -+ -+ /* Dump relevant registers */ -+ kbase_backend_gpuprops_get(kbdev, ®dump); -+ -+ gpu_props->raw_props.gpu_id = regdump.gpu_id; -+ gpu_props->raw_props.tiler_features = regdump.tiler_features; -+ gpu_props->raw_props.mem_features = regdump.mem_features; -+ gpu_props->raw_props.mmu_features = regdump.mmu_features; -+ gpu_props->raw_props.l2_features = regdump.l2_features; -+ gpu_props->raw_props.suspend_size = regdump.suspend_size; -+ -+ gpu_props->raw_props.as_present = regdump.as_present; -+ gpu_props->raw_props.js_present = regdump.js_present; -+ gpu_props->raw_props.shader_present = -+ ((u64) regdump.shader_present_hi << 32) + -+ regdump.shader_present_lo; -+ gpu_props->raw_props.tiler_present = -+ ((u64) regdump.tiler_present_hi << 32) + -+ regdump.tiler_present_lo; -+ gpu_props->raw_props.l2_present = -+ ((u64) regdump.l2_present_hi << 32) + -+ regdump.l2_present_lo; -+#ifdef CONFIG_MALI_CORESTACK -+ gpu_props->raw_props.stack_present = -+ ((u64) regdump.stack_present_hi << 32) + -+ regdump.stack_present_lo; -+#else /* CONFIG_MALI_CORESTACK */ -+ gpu_props->raw_props.stack_present = 0; -+#endif /* CONFIG_MALI_CORESTACK */ -+ -+ for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) -+ gpu_props->raw_props.js_features[i] = regdump.js_features[i]; -+ -+ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) -+ gpu_props->raw_props.texture_features[i] = regdump.texture_features[i]; -+ -+ gpu_props->raw_props.thread_max_barrier_size = regdump.thread_max_barrier_size; -+ gpu_props->raw_props.thread_max_threads = regdump.thread_max_threads; -+ gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size; -+ gpu_props->raw_props.thread_features = regdump.thread_features; -+} -+ -+void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props) -+{ -+ gpu_props->core_props.version_status = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4); -+ gpu_props->core_props.minor_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 4U, 8); -+ gpu_props->core_props.major_revision = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 12U, 4); -+ gpu_props->core_props.product_id = KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); -+} -+ -+/** -+ * kbase_gpuprops_calculate_props - Calculate the derived properties -+ * @gpu_props: The &base_gpu_props structure -+ * @kbdev: The &struct kbase_device structure for the device -+ * -+ * Fill the &base_gpu_props structure with values derived from the GPU -+ * configuration registers -+ */ -+static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev) -+{ -+ int i; -+ -+ /* Populate the base_gpu_props structure */ -+ kbase_gpuprops_update_core_props_gpu_id(gpu_props); -+ gpu_props->core_props.log2_program_counter_size = KBASE_GPU_PC_SIZE_LOG2; -+ gpu_props->core_props.gpu_available_memory_size = totalram_pages() << PAGE_SHIFT; -+ -+ for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) -+ gpu_props->core_props.texture_features[i] = gpu_props->raw_props.texture_features[i]; -+ -+ gpu_props->l2_props.log2_line_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 0U, 8); -+ gpu_props->l2_props.log2_cache_size = KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); -+ -+ /* Field with number of l2 slices is added to MEM_FEATURES register -+ * since t76x. Below code assumes that for older GPU reserved bits will -+ * be read as zero. */ -+ gpu_props->l2_props.num_l2_slices = -+ KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; -+ -+ gpu_props->tiler_props.bin_size_bytes = 1 << KBASE_UBFX32(gpu_props->raw_props.tiler_features, 0U, 6); -+ gpu_props->tiler_props.max_active_levels = KBASE_UBFX32(gpu_props->raw_props.tiler_features, 8U, 4); -+ -+ if (gpu_props->raw_props.thread_max_threads == 0) -+ gpu_props->thread_props.max_threads = THREAD_MT_DEFAULT; -+ else -+ gpu_props->thread_props.max_threads = gpu_props->raw_props.thread_max_threads; -+ -+ if (gpu_props->raw_props.thread_max_workgroup_size == 0) -+ gpu_props->thread_props.max_workgroup_size = THREAD_MWS_DEFAULT; -+ else -+ gpu_props->thread_props.max_workgroup_size = gpu_props->raw_props.thread_max_workgroup_size; -+ -+ if (gpu_props->raw_props.thread_max_barrier_size == 0) -+ gpu_props->thread_props.max_barrier_size = THREAD_MBS_DEFAULT; -+ else -+ gpu_props->thread_props.max_barrier_size = gpu_props->raw_props.thread_max_barrier_size; -+ -+ gpu_props->thread_props.max_registers = KBASE_UBFX32(gpu_props->raw_props.thread_features, 0U, 16); -+ gpu_props->thread_props.max_task_queue = KBASE_UBFX32(gpu_props->raw_props.thread_features, 16U, 8); -+ gpu_props->thread_props.max_thread_group_split = KBASE_UBFX32(gpu_props->raw_props.thread_features, 24U, 6); -+ gpu_props->thread_props.impl_tech = KBASE_UBFX32(gpu_props->raw_props.thread_features, 30U, 2); -+ -+ /* If values are not specified, then use defaults */ -+ if (gpu_props->thread_props.max_registers == 0) { -+ gpu_props->thread_props.max_registers = THREAD_MR_DEFAULT; -+ gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; -+ gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; -+ } -+ /* Initialize the coherent_group structure for each group */ -+ kbase_gpuprops_construct_coherent_groups(gpu_props); -+} -+ -+void kbase_gpuprops_set(struct kbase_device *kbdev) -+{ -+ struct kbase_gpu_props *gpu_props; -+ struct gpu_raw_gpu_props *raw; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ gpu_props = &kbdev->gpu_props; -+ raw = &gpu_props->props.raw_props; -+ -+ /* Initialize the base_gpu_props structure from the hardware */ -+ kbase_gpuprops_get_props(&gpu_props->props, kbdev); -+ -+ /* Populate the derived properties */ -+ kbase_gpuprops_calculate_props(&gpu_props->props, kbdev); -+ -+ /* Populate kbase-only fields */ -+ gpu_props->l2_props.associativity = KBASE_UBFX32(raw->l2_features, 8U, 8); -+ gpu_props->l2_props.external_bus_width = KBASE_UBFX32(raw->l2_features, 24U, 8); -+ -+ gpu_props->mem.core_group = KBASE_UBFX32(raw->mem_features, 0U, 1); -+ -+ gpu_props->mmu.va_bits = KBASE_UBFX32(raw->mmu_features, 0U, 8); -+ gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); -+ -+ gpu_props->num_cores = hweight64(raw->shader_present); -+ gpu_props->num_core_groups = hweight64(raw->l2_present); -+ gpu_props->num_address_spaces = hweight32(raw->as_present); -+ gpu_props->num_job_slots = hweight32(raw->js_present); -+} -+ -+void kbase_gpuprops_set_features(struct kbase_device *kbdev) -+{ -+ base_gpu_props *gpu_props; -+ struct kbase_gpuprops_regdump regdump; -+ -+ gpu_props = &kbdev->gpu_props.props; -+ -+ /* Dump relevant registers */ -+ kbase_backend_gpuprops_get_features(kbdev, ®dump); -+ -+ /* -+ * Copy the raw value from the register, later this will get turned -+ * into the selected coherency mode. -+ * Additionally, add non-coherent mode, as this is always supported. -+ */ -+ gpu_props->raw_props.coherency_mode = regdump.coherency_features | -+ COHERENCY_FEATURE_BIT(COHERENCY_NONE); -+} -+ -+static struct { -+ u32 type; -+ size_t offset; -+ int size; -+} gpu_property_mapping[] = { -+#define PROP(name, member) \ -+ {KBASE_GPUPROP_ ## name, offsetof(struct mali_base_gpu_props, member), \ -+ sizeof(((struct mali_base_gpu_props *)0)->member)} -+ PROP(PRODUCT_ID, core_props.product_id), -+ PROP(VERSION_STATUS, core_props.version_status), -+ PROP(MINOR_REVISION, core_props.minor_revision), -+ PROP(MAJOR_REVISION, core_props.major_revision), -+ PROP(GPU_SPEED_MHZ, core_props.gpu_speed_mhz), -+ PROP(GPU_FREQ_KHZ_MAX, core_props.gpu_freq_khz_max), -+ PROP(GPU_FREQ_KHZ_MIN, core_props.gpu_freq_khz_min), -+ PROP(LOG2_PROGRAM_COUNTER_SIZE, core_props.log2_program_counter_size), -+ PROP(TEXTURE_FEATURES_0, core_props.texture_features[0]), -+ PROP(TEXTURE_FEATURES_1, core_props.texture_features[1]), -+ PROP(TEXTURE_FEATURES_2, core_props.texture_features[2]), -+ PROP(GPU_AVAILABLE_MEMORY_SIZE, core_props.gpu_available_memory_size), -+ -+ PROP(L2_LOG2_LINE_SIZE, l2_props.log2_line_size), -+ PROP(L2_LOG2_CACHE_SIZE, l2_props.log2_cache_size), -+ PROP(L2_NUM_L2_SLICES, l2_props.num_l2_slices), -+ -+ PROP(TILER_BIN_SIZE_BYTES, tiler_props.bin_size_bytes), -+ PROP(TILER_MAX_ACTIVE_LEVELS, tiler_props.max_active_levels), -+ -+ PROP(MAX_THREADS, thread_props.max_threads), -+ PROP(MAX_WORKGROUP_SIZE, thread_props.max_workgroup_size), -+ PROP(MAX_BARRIER_SIZE, thread_props.max_barrier_size), -+ PROP(MAX_REGISTERS, thread_props.max_registers), -+ PROP(MAX_TASK_QUEUE, thread_props.max_task_queue), -+ PROP(MAX_THREAD_GROUP_SPLIT, thread_props.max_thread_group_split), -+ PROP(IMPL_TECH, thread_props.impl_tech), -+ -+ PROP(RAW_SHADER_PRESENT, raw_props.shader_present), -+ PROP(RAW_TILER_PRESENT, raw_props.tiler_present), -+ PROP(RAW_L2_PRESENT, raw_props.l2_present), -+ PROP(RAW_STACK_PRESENT, raw_props.stack_present), -+ PROP(RAW_L2_FEATURES, raw_props.l2_features), -+ PROP(RAW_SUSPEND_SIZE, raw_props.suspend_size), -+ PROP(RAW_MEM_FEATURES, raw_props.mem_features), -+ PROP(RAW_MMU_FEATURES, raw_props.mmu_features), -+ PROP(RAW_AS_PRESENT, raw_props.as_present), -+ PROP(RAW_JS_PRESENT, raw_props.js_present), -+ PROP(RAW_JS_FEATURES_0, raw_props.js_features[0]), -+ PROP(RAW_JS_FEATURES_1, raw_props.js_features[1]), -+ PROP(RAW_JS_FEATURES_2, raw_props.js_features[2]), -+ PROP(RAW_JS_FEATURES_3, raw_props.js_features[3]), -+ PROP(RAW_JS_FEATURES_4, raw_props.js_features[4]), -+ PROP(RAW_JS_FEATURES_5, raw_props.js_features[5]), -+ PROP(RAW_JS_FEATURES_6, raw_props.js_features[6]), -+ PROP(RAW_JS_FEATURES_7, raw_props.js_features[7]), -+ PROP(RAW_JS_FEATURES_8, raw_props.js_features[8]), -+ PROP(RAW_JS_FEATURES_9, raw_props.js_features[9]), -+ PROP(RAW_JS_FEATURES_10, raw_props.js_features[10]), -+ PROP(RAW_JS_FEATURES_11, raw_props.js_features[11]), -+ PROP(RAW_JS_FEATURES_12, raw_props.js_features[12]), -+ PROP(RAW_JS_FEATURES_13, raw_props.js_features[13]), -+ PROP(RAW_JS_FEATURES_14, raw_props.js_features[14]), -+ PROP(RAW_JS_FEATURES_15, raw_props.js_features[15]), -+ PROP(RAW_TILER_FEATURES, raw_props.tiler_features), -+ PROP(RAW_TEXTURE_FEATURES_0, raw_props.texture_features[0]), -+ PROP(RAW_TEXTURE_FEATURES_1, raw_props.texture_features[1]), -+ PROP(RAW_TEXTURE_FEATURES_2, raw_props.texture_features[2]), -+ PROP(RAW_GPU_ID, raw_props.gpu_id), -+ PROP(RAW_THREAD_MAX_THREADS, raw_props.thread_max_threads), -+ PROP(RAW_THREAD_MAX_WORKGROUP_SIZE, -+ raw_props.thread_max_workgroup_size), -+ PROP(RAW_THREAD_MAX_BARRIER_SIZE, raw_props.thread_max_barrier_size), -+ PROP(RAW_THREAD_FEATURES, raw_props.thread_features), -+ PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), -+ -+ PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), -+ PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), -+ PROP(COHERENCY_COHERENCY, coherency_info.coherency), -+ PROP(COHERENCY_GROUP_0, coherency_info.group[0].core_mask), -+ PROP(COHERENCY_GROUP_1, coherency_info.group[1].core_mask), -+ PROP(COHERENCY_GROUP_2, coherency_info.group[2].core_mask), -+ PROP(COHERENCY_GROUP_3, coherency_info.group[3].core_mask), -+ PROP(COHERENCY_GROUP_4, coherency_info.group[4].core_mask), -+ PROP(COHERENCY_GROUP_5, coherency_info.group[5].core_mask), -+ PROP(COHERENCY_GROUP_6, coherency_info.group[6].core_mask), -+ PROP(COHERENCY_GROUP_7, coherency_info.group[7].core_mask), -+ PROP(COHERENCY_GROUP_8, coherency_info.group[8].core_mask), -+ PROP(COHERENCY_GROUP_9, coherency_info.group[9].core_mask), -+ PROP(COHERENCY_GROUP_10, coherency_info.group[10].core_mask), -+ PROP(COHERENCY_GROUP_11, coherency_info.group[11].core_mask), -+ PROP(COHERENCY_GROUP_12, coherency_info.group[12].core_mask), -+ PROP(COHERENCY_GROUP_13, coherency_info.group[13].core_mask), -+ PROP(COHERENCY_GROUP_14, coherency_info.group[14].core_mask), -+ PROP(COHERENCY_GROUP_15, coherency_info.group[15].core_mask), -+ -+#undef PROP -+}; -+ -+int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) -+{ -+ struct kbase_gpu_props *kprops = &kbdev->gpu_props; -+ struct mali_base_gpu_props *props = &kprops->props; -+ u32 count = ARRAY_SIZE(gpu_property_mapping); -+ u32 i; -+ u32 size = 0; -+ u8 *p; -+ -+ for (i = 0; i < count; i++) { -+ /* 4 bytes for the ID, and the size of the property */ -+ size += 4 + gpu_property_mapping[i].size; -+ } -+ -+ kprops->prop_buffer_size = size; -+ kprops->prop_buffer = kmalloc(size, GFP_KERNEL); -+ -+ if (!kprops->prop_buffer) { -+ kprops->prop_buffer_size = 0; -+ return -ENOMEM; -+ } -+ -+ p = kprops->prop_buffer; -+ -+#define WRITE_U8(v) (*p++ = (v) & 0xFF) -+#define WRITE_U16(v) do { WRITE_U8(v); WRITE_U8((v) >> 8); } while (0) -+#define WRITE_U32(v) do { WRITE_U16(v); WRITE_U16((v) >> 16); } while (0) -+#define WRITE_U64(v) do { WRITE_U32(v); WRITE_U32((v) >> 32); } while (0) -+ -+ for (i = 0; i < count; i++) { -+ u32 type = gpu_property_mapping[i].type; -+ u8 type_size; -+ void *field = ((u8 *)props) + gpu_property_mapping[i].offset; -+ -+ switch (gpu_property_mapping[i].size) { -+ case 1: -+ type_size = KBASE_GPUPROP_VALUE_SIZE_U8; -+ break; -+ case 2: -+ type_size = KBASE_GPUPROP_VALUE_SIZE_U16; -+ break; -+ case 4: -+ type_size = KBASE_GPUPROP_VALUE_SIZE_U32; -+ break; -+ case 8: -+ type_size = KBASE_GPUPROP_VALUE_SIZE_U64; -+ break; -+ default: -+ dev_err(kbdev->dev, -+ "Invalid gpu_property_mapping type=%d size=%d", -+ type, gpu_property_mapping[i].size); -+ return -EINVAL; -+ } -+ -+ WRITE_U32((type<<2) | type_size); -+ -+ switch (type_size) { -+ case KBASE_GPUPROP_VALUE_SIZE_U8: -+ WRITE_U8(*((u8 *)field)); -+ break; -+ case KBASE_GPUPROP_VALUE_SIZE_U16: -+ WRITE_U16(*((u16 *)field)); -+ break; -+ case KBASE_GPUPROP_VALUE_SIZE_U32: -+ WRITE_U32(*((u32 *)field)); -+ break; -+ case KBASE_GPUPROP_VALUE_SIZE_U64: -+ WRITE_U64(*((u64 *)field)); -+ break; -+ default: /* Cannot be reached */ -+ WARN_ON(1); -+ return -EINVAL; -+ } -+ } -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h -new file mode 100755 -index 000000000..57b3eaf9c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h -@@ -0,0 +1,84 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_gpuprops.h -+ * Base kernel property query APIs -+ */ -+ -+#ifndef _KBASE_GPUPROPS_H_ -+#define _KBASE_GPUPROPS_H_ -+ -+#include "mali_kbase_gpuprops_types.h" -+ -+/* Forward definition - see mali_kbase.h */ -+struct kbase_device; -+ -+/** -+ * @brief Set up Kbase GPU properties. -+ * -+ * Set up Kbase GPU properties with information from the GPU registers -+ * -+ * @param kbdev The struct kbase_device structure for the device -+ */ -+void kbase_gpuprops_set(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpuprops_set_features - Set up Kbase GPU properties -+ * @kbdev: Device pointer -+ * -+ * This function sets up GPU properties that are dependent on the hardware -+ * features bitmask. This function must be preceeded by a call to -+ * kbase_hw_set_features_mask(). -+ */ -+void kbase_gpuprops_set_features(struct kbase_device *kbdev); -+ -+/** -+ * @brief Provide GPU properties to userside through UKU call. -+ * -+ * Fill the struct kbase_uk_gpuprops with values from GPU configuration registers. -+ * -+ * @param kctx The struct kbase_context structure -+ * @param kbase_props A copy of the struct kbase_uk_gpuprops structure from userspace -+ * -+ * @return 0 on success. Any other value indicates failure. -+ */ -+int kbase_gpuprops_uk_get_props(struct kbase_context *kctx, struct kbase_uk_gpuprops * const kbase_props); -+ -+/** -+ * kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer -+ * @kbdev: The kbase device -+ * -+ * Fills kbdev->gpu_props->prop_buffer with the GPU properties for user -+ * space to read. -+ */ -+int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpuprops_update_core_props_gpu_id - break down gpu id value -+ * @gpu_props: the &base_gpu_props structure -+ * -+ * Break down gpu_id value stored in base_gpu_props::raw_props.gpu_id into -+ * separate fields (version_status, minor_revision, major_revision, product_id) -+ * stored in base_gpu_props::core_props. -+ */ -+void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props); -+ -+ -+#endif /* _KBASE_GPUPROPS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h b/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h -new file mode 100755 -index 000000000..10794fc27 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h -@@ -0,0 +1,92 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_gpuprops_types.h -+ * Base kernel property query APIs -+ */ -+ -+#ifndef _KBASE_GPUPROPS_TYPES_H_ -+#define _KBASE_GPUPROPS_TYPES_H_ -+ -+#include "mali_base_kernel.h" -+ -+#define KBASE_GPU_SPEED_MHZ 123 -+#define KBASE_GPU_PC_SIZE_LOG2 24U -+ -+struct kbase_gpuprops_regdump { -+ u32 gpu_id; -+ u32 l2_features; -+ u32 suspend_size; /* API 8.2+ */ -+ u32 tiler_features; -+ u32 mem_features; -+ u32 mmu_features; -+ u32 as_present; -+ u32 js_present; -+ u32 thread_max_threads; -+ u32 thread_max_workgroup_size; -+ u32 thread_max_barrier_size; -+ u32 thread_features; -+ u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; -+ u32 js_features[GPU_MAX_JOB_SLOTS]; -+ u32 shader_present_lo; -+ u32 shader_present_hi; -+ u32 tiler_present_lo; -+ u32 tiler_present_hi; -+ u32 l2_present_lo; -+ u32 l2_present_hi; -+ u32 stack_present_lo; -+ u32 stack_present_hi; -+ u32 coherency_features; -+}; -+ -+struct kbase_gpu_cache_props { -+ u8 associativity; -+ u8 external_bus_width; -+}; -+ -+struct kbase_gpu_mem_props { -+ u8 core_group; -+}; -+ -+struct kbase_gpu_mmu_props { -+ u8 va_bits; -+ u8 pa_bits; -+}; -+ -+struct kbase_gpu_props { -+ /* kernel-only properties */ -+ u8 num_cores; -+ u8 num_core_groups; -+ u8 num_address_spaces; -+ u8 num_job_slots; -+ -+ struct kbase_gpu_cache_props l2_props; -+ -+ struct kbase_gpu_mem_props mem; -+ struct kbase_gpu_mmu_props mmu; -+ -+ /* Properties shared with userspace */ -+ base_gpu_props props; -+ -+ u32 prop_buffer_size; -+ void *prop_buffer; -+}; -+ -+#endif /* _KBASE_GPUPROPS_TYPES_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hw.c b/drivers/gpu/arm/midgard/mali_kbase_hw.c -new file mode 100755 -index 000000000..9a390d233 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hw.c -@@ -0,0 +1,453 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * Run-time work-arounds helpers -+ */ -+ -+#include -+#include -+#include -+#include "mali_kbase.h" -+#include "mali_kbase_hw.h" -+ -+void kbase_hw_set_features_mask(struct kbase_device *kbdev) -+{ -+ const enum base_hw_feature *features; -+ u32 gpu_id; -+ u32 product_id; -+ -+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; -+ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ -+ if (GPU_ID_IS_NEW_FORMAT(product_id)) { -+ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { -+ case GPU_ID2_PRODUCT_TMIX: -+ features = base_hw_features_tMIx; -+ break; -+ case GPU_ID2_PRODUCT_THEX: -+ features = base_hw_features_tHEx; -+ break; -+ case GPU_ID2_PRODUCT_TSIX: -+ features = base_hw_features_tSIx; -+ break; -+#ifdef MALI_INCLUDE_TKAX -+ case GPU_ID2_PRODUCT_TKAX: -+ features = base_hw_features_tKAx; -+ break; -+#endif /* MALI_INCLUDE_TKAX */ -+#ifdef MALI_INCLUDE_TTRX -+ case GPU_ID2_PRODUCT_TTRX: -+ features = base_hw_features_tTRx; -+ break; -+#endif /* MALI_INCLUDE_TTRX */ -+ default: -+ features = base_hw_features_generic; -+ break; -+ } -+ } else { -+ switch (product_id) { -+ case GPU_ID_PI_TFRX: -+ /* FALLTHROUGH */ -+ case GPU_ID_PI_T86X: -+ features = base_hw_features_tFxx; -+ break; -+ case GPU_ID_PI_T83X: -+ features = base_hw_features_t83x; -+ break; -+ case GPU_ID_PI_T82X: -+ features = base_hw_features_t82x; -+ break; -+ case GPU_ID_PI_T76X: -+ features = base_hw_features_t76x; -+ break; -+ case GPU_ID_PI_T72X: -+ features = base_hw_features_t72x; -+ break; -+ case GPU_ID_PI_T62X: -+ features = base_hw_features_t62x; -+ break; -+ case GPU_ID_PI_T60X: -+ features = base_hw_features_t60x; -+ break; -+ default: -+ features = base_hw_features_generic; -+ break; -+ } -+ } -+ -+ for (; *features != BASE_HW_FEATURE_END; features++) -+ set_bit(*features, &kbdev->hw_features_mask[0]); -+} -+ -+/** -+ * kbase_hw_get_issues_for_new_id - Get the hardware issues for a new GPU ID -+ * @kbdev: Device pointer -+ * -+ * Return: pointer to an array of hardware issues, terminated by -+ * BASE_HW_ISSUE_END. -+ * -+ * This function can only be used on new-format GPU IDs, i.e. those for which -+ * GPU_ID_IS_NEW_FORMAT evaluates as true. The GPU ID is read from the @kbdev. -+ * -+ * In debugging versions of the driver, unknown versions of a known GPU will -+ * be treated as the most recent known version not later than the actual -+ * version. In such circumstances, the GPU ID in @kbdev will also be replaced -+ * with the most recent known version. -+ * -+ * Note: The GPU configuration must have been read by kbase_gpuprops_get_props() -+ * before calling this function. -+ */ -+static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( -+ struct kbase_device *kbdev) -+{ -+ const enum base_hw_issue *issues = NULL; -+ -+ struct base_hw_product { -+ u32 product_model; -+ struct { -+ u32 version; -+ const enum base_hw_issue *issues; -+ } map[7]; -+ }; -+ -+ static const struct base_hw_product base_hw_products[] = { -+ {GPU_ID2_PRODUCT_TMIX, -+ {{GPU_ID2_VERSION_MAKE(0, 0, 1), -+ base_hw_issues_tMIx_r0p0_05dev0}, -+ {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, -+ {U32_MAX /* sentinel value */, NULL} } }, -+ -+ {GPU_ID2_PRODUCT_THEX, -+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, -+ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, -+ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, -+ {U32_MAX, NULL} } }, -+ -+ {GPU_ID2_PRODUCT_TSIX, -+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, -+ {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, -+ {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, -+ {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tSIx_r0p1}, -+ {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, -+ {GPU_ID2_VERSION_MAKE(1, 0, 1), base_hw_issues_tSIx_r1p0}, -+ {U32_MAX, NULL} } }, -+ -+ -+#ifdef MALI_INCLUDE_TKAX -+ {GPU_ID2_PRODUCT_TKAX, -+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tKAx_r0p0}, -+ {U32_MAX, NULL} } }, -+#endif /* MALI_INCLUDE_TKAX */ -+ -+#ifdef MALI_INCLUDE_TTRX -+ {GPU_ID2_PRODUCT_TTRX, -+ {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, -+ {U32_MAX, NULL} } }, -+#endif /* MALI_INCLUDE_TTRX */ -+ }; -+ -+ u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ const u32 product_model = gpu_id & GPU_ID2_PRODUCT_MODEL; -+ const struct base_hw_product *product = NULL; -+ size_t p; -+ -+ /* Stop when we reach the end of the products array. */ -+ for (p = 0; p < ARRAY_SIZE(base_hw_products); ++p) { -+ if (product_model == base_hw_products[p].product_model) { -+ product = &base_hw_products[p]; -+ break; -+ } -+ } -+ -+ if (product != NULL) { -+ /* Found a matching product. */ -+ const u32 version = gpu_id & GPU_ID2_VERSION; -+ u32 fallback_version = 0; -+ const enum base_hw_issue *fallback_issues = NULL; -+ size_t v; -+ -+ /* Stop when we reach the end of the map. */ -+ for (v = 0; product->map[v].version != U32_MAX; ++v) { -+ -+ if (version == product->map[v].version) { -+ /* Exact match so stop. */ -+ issues = product->map[v].issues; -+ break; -+ } -+ -+ /* Check whether this is a candidate for most recent -+ known version not later than the actual -+ version. */ -+ if ((version > product->map[v].version) && -+ (product->map[v].version >= fallback_version)) { -+ fallback_version = product->map[v].version; -+ fallback_issues = product->map[v].issues; -+ } -+ } -+ -+ if ((issues == NULL) && (fallback_issues != NULL)) { -+ /* Fall back to the issue set of the most recent known -+ version not later than the actual version. */ -+ issues = fallback_issues; -+ -+ dev_info(kbdev->dev, -+ "r%dp%d status %d is unknown; treating as r%dp%d status %d", -+ (gpu_id & GPU_ID2_VERSION_MAJOR) >> -+ GPU_ID2_VERSION_MAJOR_SHIFT, -+ (gpu_id & GPU_ID2_VERSION_MINOR) >> -+ GPU_ID2_VERSION_MINOR_SHIFT, -+ (gpu_id & GPU_ID2_VERSION_STATUS) >> -+ GPU_ID2_VERSION_STATUS_SHIFT, -+ (fallback_version & GPU_ID2_VERSION_MAJOR) >> -+ GPU_ID2_VERSION_MAJOR_SHIFT, -+ (fallback_version & GPU_ID2_VERSION_MINOR) >> -+ GPU_ID2_VERSION_MINOR_SHIFT, -+ (fallback_version & GPU_ID2_VERSION_STATUS) >> -+ GPU_ID2_VERSION_STATUS_SHIFT); -+ -+ gpu_id &= ~GPU_ID2_VERSION; -+ gpu_id |= fallback_version; -+ kbdev->gpu_props.props.raw_props.gpu_id = gpu_id; -+ -+ kbase_gpuprops_update_core_props_gpu_id(&kbdev->gpu_props.props); -+ } -+ } -+ return issues; -+} -+ -+int kbase_hw_set_issues_mask(struct kbase_device *kbdev) -+{ -+ const enum base_hw_issue *issues; -+ u32 gpu_id; -+ u32 product_id; -+ u32 impl_tech; -+ -+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; -+ product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ impl_tech = kbdev->gpu_props.props.thread_props.impl_tech; -+ -+ if (impl_tech != IMPLEMENTATION_MODEL) { -+ if (GPU_ID_IS_NEW_FORMAT(product_id)) { -+ issues = kbase_hw_get_issues_for_new_id(kbdev); -+ if (issues == NULL) { -+ dev_err(kbdev->dev, -+ "Unknown GPU ID %x", gpu_id); -+ return -EINVAL; -+ } -+ -+ /* The GPU ID might have been replaced with the last -+ known version of the same GPU. */ -+ gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -+ -+ } else { -+ switch (gpu_id) { -+ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_15DEV0): -+ issues = base_hw_issues_t60x_r0p0_15dev0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 0, GPU_ID_S_EAC): -+ issues = base_hw_issues_t60x_r0p0_eac; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T60X, 0, 1, 0): -+ issues = base_hw_issues_t60x_r0p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T62X, 0, 1, 0): -+ issues = base_hw_issues_t62x_r0p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 0): -+ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 0, 1): -+ issues = base_hw_issues_t62x_r1p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T62X, 1, 1, 0): -+ issues = base_hw_issues_t62x_r1p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 0, 1): -+ issues = base_hw_issues_t76x_r0p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 1): -+ issues = base_hw_issues_t76x_r0p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 1, 9): -+ issues = base_hw_issues_t76x_r0p1_50rel0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 2, 1): -+ issues = base_hw_issues_t76x_r0p2; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T76X, 0, 3, 1): -+ issues = base_hw_issues_t76x_r0p3; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T76X, 1, 0, 0): -+ issues = base_hw_issues_t76x_r1p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 0): -+ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 1): -+ case GPU_ID_MAKE(GPU_ID_PI_T72X, 0, 0, 2): -+ issues = base_hw_issues_t72x_r0p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 0, 0): -+ issues = base_hw_issues_t72x_r1p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T72X, 1, 1, 0): -+ issues = base_hw_issues_t72x_r1p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 1, 2): -+ issues = base_hw_issues_tFRx_r0p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 0, 2, 0): -+ issues = base_hw_issues_tFRx_r0p2; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 0): -+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 1, 0, 8): -+ issues = base_hw_issues_tFRx_r1p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_TFRX, 2, 0, 0): -+ issues = base_hw_issues_tFRx_r2p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T86X, 0, 2, 0): -+ issues = base_hw_issues_t86x_r0p2; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 0): -+ case GPU_ID_MAKE(GPU_ID_PI_T86X, 1, 0, 8): -+ issues = base_hw_issues_t86x_r1p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T86X, 2, 0, 0): -+ issues = base_hw_issues_t86x_r2p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T83X, 0, 1, 0): -+ issues = base_hw_issues_t83x_r0p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 0): -+ case GPU_ID_MAKE(GPU_ID_PI_T83X, 1, 0, 8): -+ issues = base_hw_issues_t83x_r1p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 0, 0): -+ issues = base_hw_issues_t82x_r0p0; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 0, 1, 0): -+ issues = base_hw_issues_t82x_r0p1; -+ break; -+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 0): -+ case GPU_ID_MAKE(GPU_ID_PI_T82X, 1, 0, 8): -+ issues = base_hw_issues_t82x_r1p0; -+ break; -+ default: -+ dev_err(kbdev->dev, -+ "Unknown GPU ID %x", gpu_id); -+ return -EINVAL; -+ } -+ } -+ } else { -+ /* Software model */ -+ if (GPU_ID_IS_NEW_FORMAT(product_id)) { -+ switch (gpu_id & GPU_ID2_PRODUCT_MODEL) { -+ case GPU_ID2_PRODUCT_TMIX: -+ issues = base_hw_issues_model_tMIx; -+ break; -+ case GPU_ID2_PRODUCT_THEX: -+ issues = base_hw_issues_model_tHEx; -+ break; -+ case GPU_ID2_PRODUCT_TSIX: -+ issues = base_hw_issues_model_tSIx; -+ break; -+#ifdef MALI_INCLUDE_TKAX -+ case GPU_ID2_PRODUCT_TKAX: -+ issues = base_hw_issues_model_tKAx; -+ break; -+#endif /* MALI_INCLUDE_TKAX */ -+#ifdef MALI_INCLUDE_TTRX -+ case GPU_ID2_PRODUCT_TTRX: -+ issues = base_hw_issues_model_tTRx; -+ break; -+#endif /* MALI_INCLUDE_TTRX */ -+ default: -+ dev_err(kbdev->dev, -+ "Unknown GPU ID %x", gpu_id); -+ return -EINVAL; -+ } -+ } else { -+ switch (product_id) { -+ case GPU_ID_PI_T60X: -+ issues = base_hw_issues_model_t60x; -+ break; -+ case GPU_ID_PI_T62X: -+ issues = base_hw_issues_model_t62x; -+ break; -+ case GPU_ID_PI_T72X: -+ issues = base_hw_issues_model_t72x; -+ break; -+ case GPU_ID_PI_T76X: -+ issues = base_hw_issues_model_t76x; -+ break; -+ case GPU_ID_PI_TFRX: -+ issues = base_hw_issues_model_tFRx; -+ break; -+ case GPU_ID_PI_T86X: -+ issues = base_hw_issues_model_t86x; -+ break; -+ case GPU_ID_PI_T83X: -+ issues = base_hw_issues_model_t83x; -+ break; -+ case GPU_ID_PI_T82X: -+ issues = base_hw_issues_model_t82x; -+ break; -+ default: -+ dev_err(kbdev->dev, "Unknown GPU ID %x", -+ gpu_id); -+ return -EINVAL; -+ } -+ } -+ } -+ -+ if (GPU_ID_IS_NEW_FORMAT(product_id)) { -+ dev_info(kbdev->dev, -+ "GPU identified as 0x%x arch %d.%d.%d r%dp%d status %d", -+ (gpu_id & GPU_ID2_PRODUCT_MAJOR) >> -+ GPU_ID2_PRODUCT_MAJOR_SHIFT, -+ (gpu_id & GPU_ID2_ARCH_MAJOR) >> -+ GPU_ID2_ARCH_MAJOR_SHIFT, -+ (gpu_id & GPU_ID2_ARCH_MINOR) >> -+ GPU_ID2_ARCH_MINOR_SHIFT, -+ (gpu_id & GPU_ID2_ARCH_REV) >> -+ GPU_ID2_ARCH_REV_SHIFT, -+ (gpu_id & GPU_ID2_VERSION_MAJOR) >> -+ GPU_ID2_VERSION_MAJOR_SHIFT, -+ (gpu_id & GPU_ID2_VERSION_MINOR) >> -+ GPU_ID2_VERSION_MINOR_SHIFT, -+ (gpu_id & GPU_ID2_VERSION_STATUS) >> -+ GPU_ID2_VERSION_STATUS_SHIFT); -+ } else { -+ dev_info(kbdev->dev, -+ "GPU identified as 0x%04x r%dp%d status %d", -+ (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> -+ GPU_ID_VERSION_PRODUCT_ID_SHIFT, -+ (gpu_id & GPU_ID_VERSION_MAJOR) >> -+ GPU_ID_VERSION_MAJOR_SHIFT, -+ (gpu_id & GPU_ID_VERSION_MINOR) >> -+ GPU_ID_VERSION_MINOR_SHIFT, -+ (gpu_id & GPU_ID_VERSION_STATUS) >> -+ GPU_ID_VERSION_STATUS_SHIFT); -+ } -+ -+ for (; *issues != BASE_HW_ISSUE_END; issues++) -+ set_bit(*issues, &kbdev->hw_issues_mask[0]); -+ -+ return 0; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hw.h b/drivers/gpu/arm/midgard/mali_kbase_hw.h -new file mode 100755 -index 000000000..754250ce9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hw.h -@@ -0,0 +1,65 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file -+ * Run-time work-arounds helpers -+ */ -+ -+#ifndef _KBASE_HW_H_ -+#define _KBASE_HW_H_ -+ -+#include "mali_kbase_defs.h" -+ -+/** -+ * @brief Tell whether a work-around should be enabled -+ */ -+#define kbase_hw_has_issue(kbdev, issue)\ -+ test_bit(issue, &(kbdev)->hw_issues_mask[0]) -+ -+/** -+ * @brief Tell whether a feature is supported -+ */ -+#define kbase_hw_has_feature(kbdev, feature)\ -+ test_bit(feature, &(kbdev)->hw_features_mask[0]) -+ -+/** -+ * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID -+ * @kbdev: Device pointer -+ * -+ * Return: 0 if the GPU ID was recognized, otherwise -EINVAL. -+ * -+ * The GPU ID is read from the @kbdev. -+ * -+ * In debugging versions of the driver, unknown versions of a known GPU with a -+ * new-format ID will be treated as the most recent known version not later -+ * than the actual version. In such circumstances, the GPU ID in @kbdev will -+ * also be replaced with the most recent known version. -+ * -+ * Note: The GPU configuration must have been read by -+ * kbase_gpuprops_get_props() before calling this function. -+ */ -+int kbase_hw_set_issues_mask(struct kbase_device *kbdev); -+ -+/** -+ * @brief Set the features mask depending on the GPU ID -+ */ -+void kbase_hw_set_features_mask(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_HW_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h -new file mode 100755 -index 000000000..b09be99e6 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h -@@ -0,0 +1,54 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * HW access backend common APIs -+ */ -+ -+#ifndef _KBASE_HWACCESS_BACKEND_H_ -+#define _KBASE_HWACCESS_BACKEND_H_ -+ -+/** -+ * kbase_backend_early_init - Perform any backend-specific initialization. -+ * @kbdev: Device pointer -+ * -+ * Return: 0 on success, or an error code on failure. -+ */ -+int kbase_backend_early_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_late_init - Perform any backend-specific initialization. -+ * @kbdev: Device pointer -+ * -+ * Return: 0 on success, or an error code on failure. -+ */ -+int kbase_backend_late_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_early_term - Perform any backend-specific termination. -+ * @kbdev: Device pointer -+ */ -+void kbase_backend_early_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_late_term - Perform any backend-specific termination. -+ * @kbdev: Device pointer -+ */ -+void kbase_backend_late_term(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_HWACCESS_BACKEND_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h -new file mode 100755 -index 000000000..0acf29719 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h -@@ -0,0 +1,36 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_hwaccess_gpu_defs.h -+ * HW access common definitions -+ */ -+ -+#ifndef _KBASE_HWACCESS_DEFS_H_ -+#define _KBASE_HWACCESS_DEFS_H_ -+ -+#include -+ -+/* The hwaccess_lock (a spinlock) must be held when accessing this structure */ -+struct kbase_hwaccess_data { -+ struct kbase_context *active_kctx; -+ -+ struct kbase_backend_data backend; -+}; -+ -+#endif /* _KBASE_HWACCESS_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h -new file mode 100755 -index 000000000..cf8a8131c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h -@@ -0,0 +1,47 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/** -+ * Base kernel property query backend APIs -+ */ -+ -+#ifndef _KBASE_HWACCESS_GPUPROPS_H_ -+#define _KBASE_HWACCESS_GPUPROPS_H_ -+ -+/** -+ * kbase_backend_gpuprops_get() - Fill @regdump with GPU properties read from -+ * GPU -+ * @kbdev: Device pointer -+ * @regdump: Pointer to struct kbase_gpuprops_regdump structure -+ */ -+void kbase_backend_gpuprops_get(struct kbase_device *kbdev, -+ struct kbase_gpuprops_regdump *regdump); -+ -+/** -+ * kbase_backend_gpuprops_get - Fill @regdump with GPU properties read from GPU -+ * @kbdev: Device pointer -+ * @regdump: Pointer to struct kbase_gpuprops_regdump structure -+ * -+ * This function reads GPU properties that are dependent on the hardware -+ * features bitmask -+ */ -+void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, -+ struct kbase_gpuprops_regdump *regdump); -+ -+ -+#endif /* _KBASE_HWACCESS_GPUPROPS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h -new file mode 100755 -index 000000000..5de2b7535 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h -@@ -0,0 +1,116 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* -+ * HW Access instrumentation common APIs -+ */ -+ -+#ifndef _KBASE_HWACCESS_INSTR_H_ -+#define _KBASE_HWACCESS_INSTR_H_ -+ -+#include -+ -+/** -+ * kbase_instr_hwcnt_enable_internal - Enable HW counters collection -+ * @kbdev: Kbase device -+ * @kctx: Kbase context -+ * @setup: HW counter setup parameters -+ * -+ * Context: might sleep, waiting for reset to complete -+ * -+ * Return: 0 on success -+ */ -+int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ struct kbase_uk_hwcnt_setup *setup); -+ -+/** -+ * kbase_instr_hwcnt_disable_internal - Disable HW counters collection -+ * @kctx: Kbase context -+ * -+ * Context: might sleep, waiting for an ongoing dump to complete -+ * -+ * Return: 0 on success -+ */ -+int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx); -+ -+/** -+ * kbase_instr_hwcnt_request_dump() - Request HW counter dump from GPU -+ * @kctx: Kbase context -+ * -+ * Caller must either wait for kbase_instr_hwcnt_dump_complete() to return true, -+ * of call kbase_instr_hwcnt_wait_for_dump(). -+ * -+ * Return: 0 on success -+ */ -+int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx); -+ -+/** -+ * kbase_instr_hwcnt_wait_for_dump() - Wait until pending HW counter dump has -+ * completed. -+ * @kctx: Kbase context -+ * -+ * Context: will sleep, waiting for dump to complete -+ * -+ * Return: 0 on success -+ */ -+int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx); -+ -+/** -+ * kbase_instr_hwcnt_dump_complete - Tell whether the HW counters dump has -+ * completed -+ * @kctx: Kbase context -+ * @success: Set to true if successful -+ * -+ * Context: does not sleep. -+ * -+ * Return: true if the dump is complete -+ */ -+bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, -+ bool * const success); -+ -+/** -+ * kbase_instr_hwcnt_clear() - Clear HW counters -+ * @kctx: Kbase context -+ * -+ * Context: might sleep, waiting for reset to complete -+ * -+ * Return: 0 on success -+ */ -+int kbase_instr_hwcnt_clear(struct kbase_context *kctx); -+ -+/** -+ * kbase_instr_backend_init() - Initialise the instrumentation backend -+ * @kbdev: Kbase device -+ * -+ * This function should be called during driver initialization. -+ * -+ * Return: 0 on success -+ */ -+int kbase_instr_backend_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_instr_backend_init() - Terminate the instrumentation backend -+ * @kbdev: Kbase device -+ * -+ * This function should be called during driver termination. -+ */ -+void kbase_instr_backend_term(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_HWACCESS_INSTR_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h -new file mode 100755 -index 000000000..750fda2cd ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h -@@ -0,0 +1,381 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * HW access job manager common APIs -+ */ -+ -+#ifndef _KBASE_HWACCESS_JM_H_ -+#define _KBASE_HWACCESS_JM_H_ -+ -+/** -+ * kbase_backend_run_atom() - Run an atom on the GPU -+ * @kbdev: Device pointer -+ * @atom: Atom to run -+ * -+ * Caller must hold the HW access lock -+ */ -+void kbase_backend_run_atom(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_backend_slot_update - Update state based on slot ringbuffers -+ * -+ * @kbdev: Device pointer -+ * -+ * Inspect the jobs in the slot ringbuffers and update state. -+ * -+ * This will cause jobs to be submitted to hardware if they are unblocked -+ */ -+void kbase_backend_slot_update(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_find_and_release_free_address_space() - Release a free AS -+ * @kbdev: Device pointer -+ * @kctx: Context pointer -+ * -+ * This function can evict an idle context from the runpool, freeing up the -+ * address space it was using. -+ * -+ * The address space is marked as in use. The caller must either assign a -+ * context using kbase_gpu_use_ctx(), or release it using -+ * kbase_ctx_sched_release() -+ * -+ * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none -+ * available -+ */ -+int kbase_backend_find_and_release_free_address_space( -+ struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the -+ * provided address space. -+ * @kbdev: Device pointer -+ * @kctx: Context pointer. May be NULL -+ * @as_nr: Free address space to use -+ * -+ * kbase_gpu_next_job() will pull atoms from the active context. -+ * -+ * Return: true if successful, false if ASID not assigned. -+ */ -+bool kbase_backend_use_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int as_nr); -+ -+/** -+ * kbase_backend_use_ctx_sched() - Activate a context. -+ * @kbdev: Device pointer -+ * @kctx: Context pointer -+ * -+ * kbase_gpu_next_job() will pull atoms from the active context. -+ * -+ * The context must already be scheduled and assigned to an address space. If -+ * the context is not scheduled, then kbase_gpu_use_ctx() should be used -+ * instead. -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if context is now active, false otherwise (ie if context does -+ * not have an address space assigned) -+ */ -+bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, -+ struct kbase_context *kctx); -+ -+/** -+ * kbase_backend_release_ctx_irq - Release a context from the GPU. This will -+ * de-assign the assigned address space. -+ * @kbdev: Device pointer -+ * @kctx: Context pointer -+ * -+ * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock -+ */ -+void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, -+ struct kbase_context *kctx); -+ -+/** -+ * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will -+ * de-assign the assigned address space. -+ * @kbdev: Device pointer -+ * @kctx: Context pointer -+ * -+ * Caller must hold kbase_device->mmu_hw_mutex -+ * -+ * This function must perform any operations that could not be performed in IRQ -+ * context by kbase_backend_release_ctx_irq(). -+ */ -+void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, -+ struct kbase_context *kctx); -+ -+/** -+ * kbase_backend_cacheclean - Perform a cache clean if the given atom requires -+ * one -+ * @kbdev: Device pointer -+ * @katom: Pointer to the failed atom -+ * -+ * On some GPUs, the GPU cache must be cleaned following a failed atom. This -+ * function performs a clean if it is required by @katom. -+ */ -+void kbase_backend_cacheclean(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+ -+/** -+ * kbase_backend_complete_wq() - Perform backend-specific actions required on -+ * completing an atom. -+ * @kbdev: Device pointer -+ * @katom: Pointer to the atom to complete -+ * -+ * This function should only be called from kbase_jd_done_worker() or -+ * js_return_worker(). -+ * -+ * Return: true if atom has completed, false if atom should be re-submitted -+ */ -+void kbase_backend_complete_wq(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_backend_complete_wq_post_sched - Perform backend-specific actions -+ * required on completing an atom, after -+ * any scheduling has taken place. -+ * @kbdev: Device pointer -+ * @core_req: Core requirements of atom -+ * @affinity: Affinity of atom -+ * @coreref_state: Coreref state of atom -+ * -+ * This function should only be called from kbase_jd_done_worker() or -+ * js_return_worker(). -+ */ -+void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, -+ base_jd_core_req core_req, u64 affinity, -+ enum kbase_atom_coreref_state coreref_state); -+ -+/** -+ * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU -+ * and remove any others from the ringbuffers. -+ * @kbdev: Device pointer -+ * @end_timestamp: Timestamp of reset -+ */ -+void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); -+ -+/** -+ * kbase_backend_inspect_head() - Return the atom currently at the head of slot -+ * @js -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * -+ * Return : Atom currently at the head of slot @js, or NULL -+ */ -+struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, -+ int js); -+ -+/** -+ * kbase_backend_inspect_tail - Return the atom currently at the tail of slot -+ * @js -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * -+ * Return : Atom currently at the head of slot @js, or NULL -+ */ -+struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, -+ int js); -+ -+/** -+ * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a -+ * slot. -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * -+ * Return : Number of atoms currently on slot -+ */ -+int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js); -+ -+/** -+ * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot -+ * that are currently on the GPU. -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * -+ * Return : Number of atoms currently on slot @js that are currently on the GPU. -+ */ -+int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js); -+ -+/** -+ * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs -+ * has changed. -+ * @kbdev: Device pointer -+ * -+ * Perform any required backend-specific actions (eg starting/stopping -+ * scheduling timers). -+ */ -+void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. -+ * @kbdev: Device pointer -+ * -+ * Perform any required backend-specific actions (eg updating timeouts of -+ * currently running atoms). -+ */ -+void kbase_backend_timeouts_changed(struct kbase_device *kbdev); -+ -+/** -+ * kbase_backend_slot_free() - Return the number of jobs that can be currently -+ * submitted to slot @js. -+ * @kbdev: Device pointer -+ * @js: Job slot to inspect -+ * -+ * Return : Number of jobs that can be submitted. -+ */ -+int kbase_backend_slot_free(struct kbase_device *kbdev, int js); -+ -+/** -+ * kbase_job_check_enter_disjoint - potentially leave disjoint state -+ * @kbdev: kbase device -+ * @target_katom: atom which is finishing -+ * -+ * Work out whether to leave disjoint state when finishing an atom that was -+ * originated by kbase_job_check_enter_disjoint(). -+ */ -+void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, -+ struct kbase_jd_atom *target_katom); -+ -+/** -+ * kbase_backend_jm_kill_jobs_from_kctx - Kill all jobs that are currently -+ * running from a context -+ * @kctx: Context pointer -+ * -+ * This is used in response to a page fault to remove all jobs from the faulting -+ * context from the hardware. -+ */ -+void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx); -+ -+/** -+ * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and -+ * to be descheduled. -+ * @kctx: Context pointer -+ * -+ * This should be called following kbase_js_zap_context(), to ensure the context -+ * can be safely destroyed. -+ */ -+void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); -+ -+/** -+ * kbase_backend_get_current_flush_id - Return the current flush ID -+ * -+ * @kbdev: Device pointer -+ * -+ * Return: the current flush ID to be recorded for each job chain -+ */ -+u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); -+ -+#if KBASE_GPU_RESET_EN -+/** -+ * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. -+ * @kbdev: Device pointer -+ * -+ * This function just soft-stops all the slots to ensure that as many jobs as -+ * possible are saved. -+ * -+ * Return: a boolean which should be interpreted as follows: -+ * - true - Prepared for reset, kbase_reset_gpu should be called. -+ * - false - Another thread is performing a reset, kbase_reset_gpu should -+ * not be called. -+ */ -+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu - Reset the GPU -+ * @kbdev: Device pointer -+ * -+ * This function should be called after kbase_prepare_to_reset_gpu if it returns -+ * true. It should never be called without a corresponding call to -+ * kbase_prepare_to_reset_gpu. -+ * -+ * After this function is called (or not called if kbase_prepare_to_reset_gpu -+ * returned false), the caller should wait for kbdev->reset_waitq to be -+ * signalled to know when the reset has completed. -+ */ -+void kbase_reset_gpu(struct kbase_device *kbdev); -+ -+/** -+ * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. -+ * @kbdev: Device pointer -+ * -+ * This function just soft-stops all the slots to ensure that as many jobs as -+ * possible are saved. -+ * -+ * Return: a boolean which should be interpreted as follows: -+ * - true - Prepared for reset, kbase_reset_gpu should be called. -+ * - false - Another thread is performing a reset, kbase_reset_gpu should -+ * not be called. -+ */ -+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_locked - Reset the GPU -+ * @kbdev: Device pointer -+ * -+ * This function should be called after kbase_prepare_to_reset_gpu if it -+ * returns true. It should never be called without a corresponding call to -+ * kbase_prepare_to_reset_gpu. -+ * -+ * After this function is called (or not called if kbase_prepare_to_reset_gpu -+ * returned false), the caller should wait for kbdev->reset_waitq to be -+ * signalled to know when the reset has completed. -+ */ -+void kbase_reset_gpu_locked(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_silent - Reset the GPU silently -+ * @kbdev: Device pointer -+ * -+ * Reset the GPU without trying to cancel jobs and don't emit messages into -+ * the kernel log while doing the reset. -+ * -+ * This function should be used in cases where we are doing a controlled reset -+ * of the GPU as part of normal processing (e.g. exiting protected mode) where -+ * the driver will have ensured the scheduler has been idled and all other -+ * users of the GPU (e.g. instrumentation) have been suspended. -+ */ -+void kbase_reset_gpu_silent(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_active - Reports if the GPU is being reset -+ * @kbdev: Device pointer -+ * -+ * Return: True if the GPU is in the process of being reset. -+ */ -+bool kbase_reset_gpu_active(struct kbase_device *kbdev); -+#endif -+ -+/** -+ * kbase_job_slot_hardstop - Hard-stop the specified job slot -+ * @kctx: The kbase context that contains the job(s) that should -+ * be hard-stopped -+ * @js: The job slot to hard-stop -+ * @target_katom: The job that should be hard-stopped (or NULL for all -+ * jobs from the context) -+ * Context: -+ * The job slot lock must be held when calling this function. -+ */ -+void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, -+ struct kbase_jd_atom *target_katom); -+ -+extern struct protected_mode_ops kbase_native_protected_ops; -+ -+#endif /* _KBASE_HWACCESS_JM_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h -new file mode 100755 -index 000000000..71c7d495c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h -@@ -0,0 +1,209 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_hwaccess_pm.h -+ * HW access power manager common APIs -+ */ -+ -+#ifndef _KBASE_HWACCESS_PM_H_ -+#define _KBASE_HWACCESS_PM_H_ -+ -+#include -+#include -+ -+#include -+ -+/* Forward definition - see mali_kbase.h */ -+struct kbase_device; -+ -+/* Functions common to all HW access backends */ -+ -+/** -+ * Initialize the power management framework. -+ * -+ * Must be called before any other power management function -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ * -+ * @return 0 if the power management framework was successfully -+ * initialized. -+ */ -+int kbase_hwaccess_pm_init(struct kbase_device *kbdev); -+ -+/** -+ * Terminate the power management framework. -+ * -+ * No power management functions may be called after this (except -+ * @ref kbase_pm_init) -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ */ -+void kbase_hwaccess_pm_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_hwaccess_pm_powerup - Power up the GPU. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @flags: Flags to pass on to kbase_pm_init_hw -+ * -+ * Power up GPU after all modules have been initialized and interrupt handlers -+ * installed. -+ * -+ * Return: 0 if powerup was successful. -+ */ -+int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, -+ unsigned int flags); -+ -+/** -+ * Halt the power management framework. -+ * -+ * Should ensure that no new interrupts are generated, but allow any currently -+ * running interrupt handlers to complete successfully. The GPU is forced off by -+ * the time this function returns, regardless of whether or not the active power -+ * policy asks for the GPU to be powered off. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ */ -+void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); -+ -+/** -+ * Perform any backend-specific actions to suspend the GPU -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ */ -+void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); -+ -+/** -+ * Perform any backend-specific actions to resume the GPU from a suspend -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ */ -+void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); -+ -+/** -+ * Perform any required actions for activating the GPU. Called when the first -+ * context goes active. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ */ -+void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); -+ -+/** -+ * Perform any required actions for idling the GPU. Called when the last -+ * context goes idle. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ */ -+void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); -+ -+ -+/** -+ * Set the debug core mask. -+ * -+ * This determines which cores the power manager is allowed to use. -+ * -+ * @param kbdev The kbase device structure for the device (must be a -+ * valid pointer) -+ * @param new_core_mask_js0 The core mask to use for job slot 0 -+ * @param new_core_mask_js0 The core mask to use for job slot 1 -+ * @param new_core_mask_js0 The core mask to use for job slot 2 -+ */ -+void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, -+ u64 new_core_mask_js0, u64 new_core_mask_js1, -+ u64 new_core_mask_js2); -+ -+ -+/** -+ * Get the current policy. -+ * -+ * Returns the policy that is currently active. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ * -+ * @return The current policy -+ */ -+const struct kbase_pm_ca_policy -+*kbase_pm_ca_get_policy(struct kbase_device *kbdev); -+ -+/** -+ * Change the policy to the one specified. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ * @param policy The policy to change to (valid pointer returned from -+ * @ref kbase_pm_ca_list_policies) -+ */ -+void kbase_pm_ca_set_policy(struct kbase_device *kbdev, -+ const struct kbase_pm_ca_policy *policy); -+ -+/** -+ * Retrieve a static list of the available policies. -+ * -+ * @param[out] policies An array pointer to take the list of policies. This may -+ * be NULL. The contents of this array must not be -+ * modified. -+ * -+ * @return The number of policies -+ */ -+int -+kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); -+ -+ -+/** -+ * Get the current policy. -+ * -+ * Returns the policy that is currently active. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ * -+ * @return The current policy -+ */ -+const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); -+ -+/** -+ * Change the policy to the one specified. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid -+ * pointer) -+ * @param policy The policy to change to (valid pointer returned from -+ * @ref kbase_pm_list_policies) -+ */ -+void kbase_pm_set_policy(struct kbase_device *kbdev, -+ const struct kbase_pm_policy *policy); -+ -+/** -+ * Retrieve a static list of the available policies. -+ * -+ * @param[out] policies An array pointer to take the list of policies. This may -+ * be NULL. The contents of this array must not be -+ * modified. -+ * -+ * @return The number of policies -+ */ -+int kbase_pm_list_policies(const struct kbase_pm_policy * const **policies); -+ -+#endif /* _KBASE_HWACCESS_PM_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h -new file mode 100755 -index 000000000..10b65798e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h -@@ -0,0 +1,53 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/** -+ * -+ */ -+ -+#ifndef _KBASE_BACKEND_TIME_H_ -+#define _KBASE_BACKEND_TIME_H_ -+ -+/** -+ * kbase_backend_get_gpu_time() - Get current GPU time -+ * @kbdev: Device pointer -+ * @cycle_counter: Pointer to u64 to store cycle counter in -+ * @system_time: Pointer to u64 to store system time in -+ * @ts: Pointer to struct timespec64 to store current monotonic -+ * time in -+ */ -+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -+ u64 *system_time, struct timespec64 *ts); -+ -+/** -+ * kbase_wait_write_flush() - Wait for GPU write flush -+ * @kctx: Context pointer -+ * -+ * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush -+ * its write buffer. -+ * -+ * If GPU resets occur then the counters are reset to zero, the delay may not be -+ * as expected. -+ * -+ * This function is only in use for BASE_HW_ISSUE_6367 -+ */ -+#ifndef CONFIG_MALI_NO_MALI -+void kbase_wait_write_flush(struct kbase_context *kctx); -+#endif -+ -+#endif /* _KBASE_BACKEND_TIME_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h b/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -new file mode 100755 -index 000000000..cf7bf1b35 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -@@ -0,0 +1,66 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_HWCNT_READER_H_ -+#define _KBASE_HWCNT_READER_H_ -+ -+/* The ids of ioctl commands. */ -+#define KBASE_HWCNT_READER 0xBE -+#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) -+#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) -+#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) -+#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) -+#define KBASE_HWCNT_READER_GET_BUFFER _IOR(KBASE_HWCNT_READER, 0x20,\ -+ struct kbase_hwcnt_reader_metadata) -+#define KBASE_HWCNT_READER_PUT_BUFFER _IOW(KBASE_HWCNT_READER, 0x21,\ -+ struct kbase_hwcnt_reader_metadata) -+#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) -+#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) -+#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) -+#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) -+ -+/** -+ * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata -+ * @timestamp: time when sample was collected -+ * @event_id: id of an event that triggered sample collection -+ * @buffer_idx: position in sampling area where sample buffer was stored -+ */ -+struct kbase_hwcnt_reader_metadata { -+ u64 timestamp; -+ u32 event_id; -+ u32 buffer_idx; -+}; -+ -+/** -+ * enum base_hwcnt_reader_event - hwcnt dumping events -+ * @BASE_HWCNT_READER_EVENT_MANUAL: manual request for dump -+ * @BASE_HWCNT_READER_EVENT_PERIODIC: periodic dump -+ * @BASE_HWCNT_READER_EVENT_PREJOB: prejob dump request -+ * @BASE_HWCNT_READER_EVENT_POSTJOB: postjob dump request -+ * @BASE_HWCNT_READER_EVENT_COUNT: number of supported events -+ */ -+enum base_hwcnt_reader_event { -+ BASE_HWCNT_READER_EVENT_MANUAL, -+ BASE_HWCNT_READER_EVENT_PERIODIC, -+ BASE_HWCNT_READER_EVENT_PREJOB, -+ BASE_HWCNT_READER_EVENT_POSTJOB, -+ -+ BASE_HWCNT_READER_EVENT_COUNT -+}; -+ -+#endif /* _KBASE_HWCNT_READER_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_ioctl.h b/drivers/gpu/arm/midgard/mali_kbase_ioctl.h -new file mode 100755 -index 000000000..dcbed9c77 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_ioctl.h -@@ -0,0 +1,656 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_IOCTL_H_ -+#define _KBASE_IOCTL_H_ -+ -+#ifdef __cpluscplus -+extern "C" { -+#endif -+ -+#include -+ -+#define KBASE_IOCTL_TYPE 0x80 -+ -+#ifdef ANDROID -+/* Android's definition of ioctl is incorrect, specifying the type argument as -+ * 'int'. This creates a warning when using _IOWR (as the top bit is set). Work -+ * round this by redefining _IOC to include a case to 'int'. -+ */ -+#undef _IOC -+#define _IOC(dir, type, nr, size) \ -+ ((int)(((dir) << _IOC_DIRSHIFT) | ((type) << _IOC_TYPESHIFT) | \ -+ ((nr) << _IOC_NRSHIFT) | ((size) << _IOC_SIZESHIFT))) -+#endif -+ -+/** -+ * struct kbase_ioctl_version_check - Check version compatibility with kernel -+ * -+ * @major: Major version number -+ * @minor: Minor version number -+ */ -+struct kbase_ioctl_version_check { -+ __u16 major; -+ __u16 minor; -+}; -+ -+#define KBASE_IOCTL_VERSION_CHECK \ -+ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) -+ -+/** -+ * struct kbase_ioctl_set_flags - Set kernel context creation flags -+ * -+ * @create_flags: Flags - see base_context_create_flags -+ */ -+struct kbase_ioctl_set_flags { -+ __u32 create_flags; -+}; -+ -+#define KBASE_IOCTL_SET_FLAGS \ -+ _IOW(KBASE_IOCTL_TYPE, 1, struct kbase_ioctl_set_flags) -+ -+/** -+ * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel -+ * -+ * @addr: Memory address of an array of struct base_jd_atom_v2 -+ * @nr_atoms: Number of entries in the array -+ * @stride: sizeof(struct base_jd_atom_v2) -+ */ -+struct kbase_ioctl_job_submit { -+ union kbase_pointer addr; -+ __u32 nr_atoms; -+ __u32 stride; -+}; -+ -+#define KBASE_IOCTL_JOB_SUBMIT \ -+ _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) -+ -+/** -+ * struct kbase_ioctl_get_gpuprops - Read GPU properties from the kernel -+ * -+ * @buffer: Pointer to the buffer to store properties into -+ * @size: Size of the buffer -+ * @flags: Flags - must be zero for now -+ * -+ * The ioctl will return the number of bytes stored into @buffer or an error -+ * on failure (e.g. @size is too small). If @size is specified as 0 then no -+ * data will be written but the return value will be the number of bytes needed -+ * for all the properties. -+ * -+ * @flags may be used in the future to request a different format for the -+ * buffer. With @flags == 0 the following format is used. -+ * -+ * The buffer will be filled with pairs of values, a u32 key identifying the -+ * property followed by the value. The size of the value is identified using -+ * the bottom bits of the key. The value then immediately followed the key and -+ * is tightly packed (there is no padding). All keys and values are -+ * little-endian. -+ * -+ * 00 = u8 -+ * 01 = u16 -+ * 10 = u32 -+ * 11 = u64 -+ */ -+struct kbase_ioctl_get_gpuprops { -+ union kbase_pointer buffer; -+ __u32 size; -+ __u32 flags; -+}; -+ -+#define KBASE_IOCTL_GET_GPUPROPS \ -+ _IOW(KBASE_IOCTL_TYPE, 3, struct kbase_ioctl_get_gpuprops) -+ -+#define KBASE_IOCTL_POST_TERM \ -+ _IO(KBASE_IOCTL_TYPE, 4) -+ -+/** -+ * union kbase_ioctl_mem_alloc - Allocate memory on the GPU -+ * -+ * @va_pages: The number of pages of virtual address space to reserve -+ * @commit_pages: The number of physical pages to allocate -+ * @extent: The number of extra pages to allocate on each GPU fault which grows -+ * the region -+ * @flags: Flags -+ * @gpu_va: The GPU virtual address which is allocated -+ * -+ * @in: Input parameters -+ * @out: Output parameters -+ */ -+union kbase_ioctl_mem_alloc { -+ struct { -+ __u64 va_pages; -+ __u64 commit_pages; -+ __u64 extent; -+ __u64 flags; -+ } in; -+ struct { -+ __u64 flags; -+ __u64 gpu_va; -+ } out; -+}; -+ -+#define KBASE_IOCTL_MEM_ALLOC \ -+ _IOWR(KBASE_IOCTL_TYPE, 5, union kbase_ioctl_mem_alloc) -+ -+/** -+ * struct kbase_ioctl_mem_query - Query properties of a GPU memory region -+ * @gpu_addr: A GPU address contained within the region -+ * @query: The type of query -+ * @value: The result of the query -+ * -+ * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. -+ * -+ * @in: Input parameters -+ * @out: Output parameters -+ */ -+union kbase_ioctl_mem_query { -+ struct { -+ __u64 gpu_addr; -+ __u64 query; -+ } in; -+ struct { -+ __u64 value; -+ } out; -+}; -+ -+#define KBASE_IOCTL_MEM_QUERY \ -+ _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) -+ -+#define KBASE_MEM_QUERY_COMMIT_SIZE 1 -+#define KBASE_MEM_QUERY_VA_SIZE 2 -+#define KBASE_MEM_QUERY_FLAGS 3 -+ -+/** -+ * struct kbase_ioctl_mem_free - Free a memory region -+ * @gpu_addr: Handle to the region to free -+ */ -+struct kbase_ioctl_mem_free { -+ __u64 gpu_addr; -+}; -+ -+#define KBASE_IOCTL_MEM_FREE \ -+ _IOW(KBASE_IOCTL_TYPE, 7, struct kbase_ioctl_mem_free) -+ -+/** -+ * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader -+ * @buffer_count: requested number of dumping buffers -+ * @jm_bm: counters selection bitmask (JM) -+ * @shader_bm: counters selection bitmask (Shader) -+ * @tiler_bm: counters selection bitmask (Tiler) -+ * @mmu_l2_bm: counters selection bitmask (MMU_L2) -+ * -+ * A fd is returned from the ioctl if successful, or a negative value on error -+ */ -+struct kbase_ioctl_hwcnt_reader_setup { -+ __u32 buffer_count; -+ __u32 jm_bm; -+ __u32 shader_bm; -+ __u32 tiler_bm; -+ __u32 mmu_l2_bm; -+}; -+ -+#define KBASE_IOCTL_HWCNT_READER_SETUP \ -+ _IOW(KBASE_IOCTL_TYPE, 8, struct kbase_ioctl_hwcnt_reader_setup) -+ -+/** -+ * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection -+ * @dump_buffer: GPU address to write counters to -+ * @jm_bm: counters selection bitmask (JM) -+ * @shader_bm: counters selection bitmask (Shader) -+ * @tiler_bm: counters selection bitmask (Tiler) -+ * @mmu_l2_bm: counters selection bitmask (MMU_L2) -+ */ -+struct kbase_ioctl_hwcnt_enable { -+ __u64 dump_buffer; -+ __u32 jm_bm; -+ __u32 shader_bm; -+ __u32 tiler_bm; -+ __u32 mmu_l2_bm; -+}; -+ -+#define KBASE_IOCTL_HWCNT_ENABLE \ -+ _IOW(KBASE_IOCTL_TYPE, 9, struct kbase_ioctl_hwcnt_enable) -+ -+#define KBASE_IOCTL_HWCNT_DUMP \ -+ _IO(KBASE_IOCTL_TYPE, 10) -+ -+#define KBASE_IOCTL_HWCNT_CLEAR \ -+ _IO(KBASE_IOCTL_TYPE, 11) -+ -+/** -+ * struct kbase_ioctl_disjoint_query - Query the disjoint counter -+ * @counter: A counter of disjoint events in the kernel -+ */ -+struct kbase_ioctl_disjoint_query { -+ __u32 counter; -+}; -+ -+#define KBASE_IOCTL_DISJOINT_QUERY \ -+ _IOR(KBASE_IOCTL_TYPE, 12, struct kbase_ioctl_disjoint_query) -+ -+/** -+ * struct kbase_ioctl_get_ddk_version - Query the kernel version -+ * @version_buffer: Buffer to receive the kernel version string -+ * @size: Size of the buffer -+ * -+ * The ioctl will return the number of bytes written into version_buffer -+ * (which includes a NULL byte) or a negative error code -+ */ -+struct kbase_ioctl_get_ddk_version { -+ union kbase_pointer version_buffer; -+ __u32 size; -+}; -+ -+#define KBASE_IOCTL_GET_DDK_VERSION \ -+ _IOW(KBASE_IOCTL_TYPE, 13, struct kbase_ioctl_get_ddk_version) -+ -+/** -+ * struct kbase_ioctl_mem_jit_init - Initialise the JIT memory allocator -+ * -+ * @va_pages: Number of VA pages to reserve for JIT -+ * -+ * Note that depending on the VA size of the application and GPU, the value -+ * specified in @va_pages may be ignored. -+ */ -+struct kbase_ioctl_mem_jit_init { -+ __u64 va_pages; -+}; -+ -+#define KBASE_IOCTL_MEM_JIT_INIT \ -+ _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) -+ -+/** -+ * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory -+ * -+ * @handle: GPU memory handle (GPU VA) -+ * @user_addr: The address where it is mapped in user space -+ * @size: The number of bytes to synchronise -+ * @type: The direction to synchronise: 0 is sync to memory (clean), -+ * 1 is sync from memory (invalidate). Use the BASE_SYNCSET_OP_xxx constants. -+ * @padding: Padding to round up to a multiple of 8 bytes, must be zero -+ */ -+struct kbase_ioctl_mem_sync { -+ __u64 handle; -+ __u64 user_addr; -+ __u64 size; -+ __u8 type; -+ __u8 padding[7]; -+}; -+ -+#define KBASE_IOCTL_MEM_SYNC \ -+ _IOW(KBASE_IOCTL_TYPE, 15, struct kbase_ioctl_mem_sync) -+ -+/** -+ * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer -+ * -+ * @gpu_addr: The GPU address of the memory region -+ * @cpu_addr: The CPU address to locate -+ * @size: A size in bytes to validate is contained within the region -+ * @offset: The offset from the start of the memory region to @cpu_addr -+ * -+ * @in: Input parameters -+ * @out: Output parameters -+ */ -+union kbase_ioctl_mem_find_cpu_offset { -+ struct { -+ __u64 gpu_addr; -+ __u64 cpu_addr; -+ __u64 size; -+ } in; -+ struct { -+ __u64 offset; -+ } out; -+}; -+ -+#define KBASE_IOCTL_MEM_FIND_CPU_OFFSET \ -+ _IOWR(KBASE_IOCTL_TYPE, 16, union kbase_ioctl_mem_find_cpu_offset) -+ -+/** -+ * struct kbase_ioctl_get_context_id - Get the kernel context ID -+ * -+ * @id: The kernel context ID -+ */ -+struct kbase_ioctl_get_context_id { -+ int id; /* This should really be __u32, but see GPUCORE-10048 */ -+}; -+ -+#define KBASE_IOCTL_GET_CONTEXT_ID \ -+ _IOR(KBASE_IOCTL_TYPE, 17, struct kbase_ioctl_get_context_id) -+ -+/** -+ * struct kbase_ioctl_tlstream_acquire - Acquire a tlstream fd -+ * -+ * @flags: Flags -+ * -+ * The ioctl returns a file descriptor when successful -+ */ -+struct kbase_ioctl_tlstream_acquire { -+ __u32 flags; -+}; -+ -+#define KBASE_IOCTL_TLSTREAM_ACQUIRE \ -+ _IOW(KBASE_IOCTL_TYPE, 18, struct kbase_ioctl_tlstream_acquire) -+ -+#define KBASE_IOCTL_TLSTREAM_FLUSH \ -+ _IO(KBASE_IOCTL_TYPE, 19) -+ -+/** -+ * struct kbase_ioctl_mem_commit - Change the amount of memory backing a region -+ * -+ * @gpu_addr: The memory region to modify -+ * @pages: The number of physical pages that should be present -+ * -+ * The ioctl may return on the following error codes or 0 for success: -+ * -ENOMEM: Out of memory -+ * -EINVAL: Invalid arguments -+ */ -+struct kbase_ioctl_mem_commit { -+ __u64 gpu_addr; -+ __u64 pages; -+}; -+ -+#define KBASE_IOCTL_MEM_COMMIT \ -+ _IOW(KBASE_IOCTL_TYPE, 20, struct kbase_ioctl_mem_commit) -+ -+/** -+ * union kbase_ioctl_mem_alias - Create an alias of memory regions -+ * @flags: Flags, see BASE_MEM_xxx -+ * @stride: Bytes between start of each memory region -+ * @nents: The number of regions to pack together into the alias -+ * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info -+ * @gpu_va: Address of the new alias -+ * @va_pages: Size of the new alias -+ * -+ * @in: Input parameters -+ * @out: Output parameters -+ */ -+union kbase_ioctl_mem_alias { -+ struct { -+ __u64 flags; -+ __u64 stride; -+ __u64 nents; -+ union kbase_pointer aliasing_info; -+ } in; -+ struct { -+ __u64 flags; -+ __u64 gpu_va; -+ __u64 va_pages; -+ } out; -+}; -+ -+#define KBASE_IOCTL_MEM_ALIAS \ -+ _IOWR(KBASE_IOCTL_TYPE, 21, union kbase_ioctl_mem_alias) -+ -+/** -+ * union kbase_ioctl_mem_import - Import memory for use by the GPU -+ * @flags: Flags, see BASE_MEM_xxx -+ * @phandle: Handle to the external memory -+ * @type: Type of external memory, see base_mem_import_type -+ * @padding: Amount of extra VA pages to append to the imported buffer -+ * @gpu_va: Address of the new alias -+ * @va_pages: Size of the new alias -+ * -+ * @in: Input parameters -+ * @out: Output parameters -+ */ -+union kbase_ioctl_mem_import { -+ struct { -+ __u64 flags; -+ union kbase_pointer phandle; -+ __u32 type; -+ __u32 padding; -+ } in; -+ struct { -+ __u64 flags; -+ __u64 gpu_va; -+ __u64 va_pages; -+ } out; -+}; -+ -+#define KBASE_IOCTL_MEM_IMPORT \ -+ _IOWR(KBASE_IOCTL_TYPE, 22, union kbase_ioctl_mem_import) -+ -+/** -+ * struct kbase_ioctl_mem_flags_change - Change the flags for a memory region -+ * @gpu_va: The GPU region to modify -+ * @flags: The new flags to set -+ * @mask: Mask of the flags to modify -+ */ -+struct kbase_ioctl_mem_flags_change { -+ __u64 gpu_va; -+ __u64 flags; -+ __u64 mask; -+}; -+ -+#define KBASE_IOCTL_MEM_FLAGS_CHANGE \ -+ _IOW(KBASE_IOCTL_TYPE, 23, struct kbase_ioctl_mem_flags_change) -+ -+/** -+ * struct kbase_ioctl_stream_create - Create a synchronisation stream -+ * @name: A name to identify this stream. Must be NULL-terminated. -+ * -+ * Note that this is also called a "timeline", but is named stream to avoid -+ * confusion with other uses of the word. -+ * -+ * Unused bytes in @name (after the first NULL byte) must be also be NULL bytes. -+ * -+ * The ioctl returns a file descriptor. -+ */ -+struct kbase_ioctl_stream_create { -+ char name[32]; -+}; -+ -+#define KBASE_IOCTL_STREAM_CREATE \ -+ _IOW(KBASE_IOCTL_TYPE, 24, struct kbase_ioctl_stream_create) -+ -+/** -+ * struct kbase_ioctl_fence_validate - Validate a fd refers to a fence -+ * @fd: The file descriptor to validate -+ */ -+struct kbase_ioctl_fence_validate { -+ int fd; -+}; -+ -+#define KBASE_IOCTL_FENCE_VALIDATE \ -+ _IOW(KBASE_IOCTL_TYPE, 25, struct kbase_ioctl_fence_validate) -+ -+/** -+ * struct kbase_ioctl_get_profiling_controls - Get the profiling controls -+ * @count: The size of @buffer in u32 words -+ * @buffer: The buffer to receive the profiling controls -+ */ -+struct kbase_ioctl_get_profiling_controls { -+ union kbase_pointer buffer; -+ __u32 count; -+}; -+ -+#define KBASE_IOCTL_GET_PROFILING_CONTROLS \ -+ _IOW(KBASE_IOCTL_TYPE, 26, struct kbase_ioctl_get_profiling_controls) -+ -+/** -+ * struct kbase_ioctl_mem_profile_add - Provide profiling information to kernel -+ * @buffer: Pointer to the information -+ * @len: Length -+ * @padding: Padding -+ * -+ * The data provided is accessible through a debugfs file -+ */ -+struct kbase_ioctl_mem_profile_add { -+ union kbase_pointer buffer; -+ __u32 len; -+ __u32 padding; -+}; -+ -+#define KBASE_IOCTL_MEM_PROFILE_ADD \ -+ _IOW(KBASE_IOCTL_TYPE, 27, struct kbase_ioctl_mem_profile_add) -+ -+/** -+ * struct kbase_ioctl_soft_event_update - Update the status of a soft-event -+ * @event: GPU address of the event which has been updated -+ * @new_status: The new status to set -+ * @flags: Flags for future expansion -+ */ -+struct kbase_ioctl_soft_event_update { -+ __u64 event; -+ __u32 new_status; -+ __u32 flags; -+}; -+ -+#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ -+ _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) -+ -+/*************** -+ * test ioctls * -+ ***************/ -+#if MALI_UNIT_TEST -+/* These ioctls are purely for test purposes and are not used in the production -+ * driver, they therefore may change without notice -+ */ -+ -+#define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) -+ -+/** -+ * struct kbase_ioctl_tlstream_test - Start a timeline stream test -+ * -+ * @tpw_count: number of trace point writers in each context -+ * @msg_delay: time delay between tracepoints from one writer in milliseconds -+ * @msg_count: number of trace points written by one writer -+ * @aux_msg: if non-zero aux messages will be included -+ */ -+struct kbase_ioctl_tlstream_test { -+ __u32 tpw_count; -+ __u32 msg_delay; -+ __u32 msg_count; -+ __u32 aux_msg; -+}; -+ -+#define KBASE_IOCTL_TLSTREAM_TEST \ -+ _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) -+ -+/** -+ * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes -+ * @bytes_collected: number of bytes read by user -+ * @bytes_generated: number of bytes generated by tracepoints -+ */ -+struct kbase_ioctl_tlstream_stats { -+ __u32 bytes_collected; -+ __u32 bytes_generated; -+}; -+ -+#define KBASE_IOCTL_TLSTREAM_STATS \ -+ _IOR(KBASE_IOCTL_TEST_TYPE, 2, struct kbase_ioctl_tlstream_stats) -+ -+#endif -+ -+/********************************** -+ * Definitions for GPU properties * -+ **********************************/ -+#define KBASE_GPUPROP_VALUE_SIZE_U8 (0x0) -+#define KBASE_GPUPROP_VALUE_SIZE_U16 (0x1) -+#define KBASE_GPUPROP_VALUE_SIZE_U32 (0x2) -+#define KBASE_GPUPROP_VALUE_SIZE_U64 (0x3) -+ -+#define KBASE_GPUPROP_PRODUCT_ID 1 -+#define KBASE_GPUPROP_VERSION_STATUS 2 -+#define KBASE_GPUPROP_MINOR_REVISION 3 -+#define KBASE_GPUPROP_MAJOR_REVISION 4 -+#define KBASE_GPUPROP_GPU_SPEED_MHZ 5 -+#define KBASE_GPUPROP_GPU_FREQ_KHZ_MAX 6 -+#define KBASE_GPUPROP_GPU_FREQ_KHZ_MIN 7 -+#define KBASE_GPUPROP_LOG2_PROGRAM_COUNTER_SIZE 8 -+#define KBASE_GPUPROP_TEXTURE_FEATURES_0 9 -+#define KBASE_GPUPROP_TEXTURE_FEATURES_1 10 -+#define KBASE_GPUPROP_TEXTURE_FEATURES_2 11 -+#define KBASE_GPUPROP_GPU_AVAILABLE_MEMORY_SIZE 12 -+ -+#define KBASE_GPUPROP_L2_LOG2_LINE_SIZE 13 -+#define KBASE_GPUPROP_L2_LOG2_CACHE_SIZE 14 -+#define KBASE_GPUPROP_L2_NUM_L2_SLICES 15 -+ -+#define KBASE_GPUPROP_TILER_BIN_SIZE_BYTES 16 -+#define KBASE_GPUPROP_TILER_MAX_ACTIVE_LEVELS 17 -+ -+#define KBASE_GPUPROP_MAX_THREADS 18 -+#define KBASE_GPUPROP_MAX_WORKGROUP_SIZE 19 -+#define KBASE_GPUPROP_MAX_BARRIER_SIZE 20 -+#define KBASE_GPUPROP_MAX_REGISTERS 21 -+#define KBASE_GPUPROP_MAX_TASK_QUEUE 22 -+#define KBASE_GPUPROP_MAX_THREAD_GROUP_SPLIT 23 -+#define KBASE_GPUPROP_IMPL_TECH 24 -+ -+#define KBASE_GPUPROP_RAW_SHADER_PRESENT 25 -+#define KBASE_GPUPROP_RAW_TILER_PRESENT 26 -+#define KBASE_GPUPROP_RAW_L2_PRESENT 27 -+#define KBASE_GPUPROP_RAW_STACK_PRESENT 28 -+#define KBASE_GPUPROP_RAW_L2_FEATURES 29 -+#define KBASE_GPUPROP_RAW_SUSPEND_SIZE 30 -+#define KBASE_GPUPROP_RAW_MEM_FEATURES 31 -+#define KBASE_GPUPROP_RAW_MMU_FEATURES 32 -+#define KBASE_GPUPROP_RAW_AS_PRESENT 33 -+#define KBASE_GPUPROP_RAW_JS_PRESENT 34 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_0 35 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_1 36 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_2 37 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_3 38 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_4 39 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_5 40 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_6 41 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_7 42 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_8 43 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_9 44 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_10 45 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_11 46 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_12 47 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_13 48 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_14 49 -+#define KBASE_GPUPROP_RAW_JS_FEATURES_15 50 -+#define KBASE_GPUPROP_RAW_TILER_FEATURES 51 -+#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_0 52 -+#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_1 53 -+#define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_2 54 -+#define KBASE_GPUPROP_RAW_GPU_ID 55 -+#define KBASE_GPUPROP_RAW_THREAD_MAX_THREADS 56 -+#define KBASE_GPUPROP_RAW_THREAD_MAX_WORKGROUP_SIZE 57 -+#define KBASE_GPUPROP_RAW_THREAD_MAX_BARRIER_SIZE 58 -+#define KBASE_GPUPROP_RAW_THREAD_FEATURES 59 -+#define KBASE_GPUPROP_RAW_COHERENCY_MODE 60 -+ -+#define KBASE_GPUPROP_COHERENCY_NUM_GROUPS 61 -+#define KBASE_GPUPROP_COHERENCY_NUM_CORE_GROUPS 62 -+#define KBASE_GPUPROP_COHERENCY_COHERENCY 63 -+#define KBASE_GPUPROP_COHERENCY_GROUP_0 64 -+#define KBASE_GPUPROP_COHERENCY_GROUP_1 65 -+#define KBASE_GPUPROP_COHERENCY_GROUP_2 66 -+#define KBASE_GPUPROP_COHERENCY_GROUP_3 67 -+#define KBASE_GPUPROP_COHERENCY_GROUP_4 68 -+#define KBASE_GPUPROP_COHERENCY_GROUP_5 69 -+#define KBASE_GPUPROP_COHERENCY_GROUP_6 70 -+#define KBASE_GPUPROP_COHERENCY_GROUP_7 71 -+#define KBASE_GPUPROP_COHERENCY_GROUP_8 72 -+#define KBASE_GPUPROP_COHERENCY_GROUP_9 73 -+#define KBASE_GPUPROP_COHERENCY_GROUP_10 74 -+#define KBASE_GPUPROP_COHERENCY_GROUP_11 75 -+#define KBASE_GPUPROP_COHERENCY_GROUP_12 76 -+#define KBASE_GPUPROP_COHERENCY_GROUP_13 77 -+#define KBASE_GPUPROP_COHERENCY_GROUP_14 78 -+#define KBASE_GPUPROP_COHERENCY_GROUP_15 79 -+ -+#ifdef __cpluscplus -+} -+#endif -+ -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd.c b/drivers/gpu/arm/midgard/mali_kbase_jd.c -new file mode 100755 -index 000000000..d9d8658d3 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_jd.c -@@ -0,0 +1,1903 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+#include -+#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ -+#ifdef CONFIG_COMPAT -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+ -+#include "mali_kbase_dma_fence.h" -+ -+#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) -+/* random32 was renamed to prandom_u32 in 3.8 */ -+#define prandom_u32 random32 -+#endif -+ -+/* Return whether katom will run on the GPU or not. Currently only soft jobs and -+ * dependency-only atoms do not run on the GPU */ -+#define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ -+ ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ -+ BASE_JD_REQ_DEP))) -+/* -+ * This is the kernel side of the API. Only entry points are: -+ * - kbase_jd_submit(): Called from userspace to submit a single bag -+ * - kbase_jd_done(): Called from interrupt context to track the -+ * completion of a job. -+ * Callouts: -+ * - to the job manager (enqueue a job) -+ * - to the event subsystem (signals the completion/failure of bag/job-chains). -+ */ -+ -+static void __user * -+get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p) -+{ -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ return compat_ptr(p->compat_value); -+#endif -+ return p->value; -+} -+ -+/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs -+ * -+ * Returns whether the JS needs a reschedule. -+ * -+ * Note that the caller must also check the atom status and -+ * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock -+ */ -+static int jd_run_atom(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); -+ -+ if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { -+ /* Dependency only atom */ -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ return 0; -+ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { -+ /* Soft-job */ -+ if (katom->will_fail_event_code) { -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ return 0; -+ } -+ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) -+ == BASE_JD_REQ_SOFT_REPLAY) { -+ if (!kbase_replay_process(katom)) -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ } else if (kbase_process_soft_job(katom) == 0) { -+ kbase_finish_soft_job(katom); -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ } -+ return 0; -+ } -+ -+ katom->status = KBASE_JD_ATOM_STATE_IN_JS; -+ /* Queue an action about whether we should try scheduling a context */ -+ return kbasep_js_add_job(kctx, katom); -+} -+ -+#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) -+void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) -+{ -+ struct kbase_device *kbdev; -+ -+ KBASE_DEBUG_ASSERT(katom); -+ kbdev = katom->kctx->kbdev; -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ /* Check whether the atom's other dependencies were already met. If -+ * katom is a GPU atom then the job scheduler may be able to represent -+ * the dependencies, hence we may attempt to submit it before they are -+ * met. Other atoms must have had both dependencies resolved. -+ */ -+ if (IS_GPU_ATOM(katom) || -+ (!kbase_jd_katom_dep_atom(&katom->dep[0]) && -+ !kbase_jd_katom_dep_atom(&katom->dep[1]))) { -+ /* katom dep complete, attempt to run it */ -+ bool resched = false; -+ -+ resched = jd_run_atom(katom); -+ -+ if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { -+ /* The atom has already finished */ -+ resched |= jd_done_nolock(katom, NULL); -+ } -+ -+ if (resched) -+ kbase_js_sched_all(kbdev); -+ } -+} -+#endif -+ -+#ifdef CONFIG_KDS -+ -+/* Add the katom to the kds waiting list. -+ * Atoms must be added to the waiting list after a successful call to kds_async_waitall. -+ * The caller must hold the kbase_jd_context.lock */ -+ -+static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx; -+ -+ KBASE_DEBUG_ASSERT(katom); -+ -+ kctx = katom->kctx; -+ -+ list_add_tail(&katom->node, &kctx->waiting_kds_resource); -+} -+ -+/* Remove the katom from the kds waiting list. -+ * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync. -+ * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add. -+ * The caller must hold the kbase_jd_context.lock */ -+ -+static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom) -+{ -+ KBASE_DEBUG_ASSERT(katom); -+ list_del(&katom->node); -+} -+ -+static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter) -+{ -+ struct kbase_jd_atom *katom; -+ struct kbase_jd_context *ctx; -+ -+ katom = (struct kbase_jd_atom *)callback_parameter; -+ KBASE_DEBUG_ASSERT(katom); -+ -+ ctx = &katom->kctx->jctx; -+ -+ /* If KDS resource has already been satisfied (e.g. due to zapping) -+ * do nothing. -+ */ -+ mutex_lock(&ctx->lock); -+ if (!katom->kds_dep_satisfied) { -+ katom->kds_dep_satisfied = true; -+ kbase_jd_dep_clear_locked(katom); -+ } -+ mutex_unlock(&ctx->lock); -+} -+ -+static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom) -+{ -+ KBASE_DEBUG_ASSERT(katom); -+ -+ /* Prevent job_done_nolock from being called twice on an atom when -+ * there is a race between job completion and cancellation */ -+ -+ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { -+ /* Wait was cancelled - zap the atom */ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ if (jd_done_nolock(katom, NULL)) -+ kbase_js_sched_all(katom->kctx->kbdev); -+ } -+} -+#endif /* CONFIG_KDS */ -+ -+void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) -+{ -+#ifdef CONFIG_KDS -+ if (katom->kds_rset) { -+ struct kbase_jd_context *jctx = &katom->kctx->jctx; -+ -+ /* -+ * As the atom is no longer waiting, remove it from -+ * the waiting list. -+ */ -+ -+ mutex_lock(&jctx->lock); -+ kbase_jd_kds_waiters_remove(katom); -+ mutex_unlock(&jctx->lock); -+ -+ /* Release the kds resource or cancel if zapping */ -+ kds_resource_set_release_sync(&katom->kds_rset); -+ } -+#endif /* CONFIG_KDS */ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ /* Flush dma-fence workqueue to ensure that any callbacks that may have -+ * been queued are done before continuing. -+ * Any successfully completed atom would have had all it's callbacks -+ * completed before the atom was run, so only flush for failed atoms. -+ */ -+ if (katom->event_code != BASE_JD_EVENT_DONE) -+ flush_workqueue(katom->kctx->dma_fence.wq); -+#endif /* CONFIG_MALI_DMA_FENCE */ -+} -+ -+static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) -+{ -+ KBASE_DEBUG_ASSERT(katom); -+ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); -+ -+#ifdef CONFIG_KDS -+ /* Prevent the KDS resource from triggering the atom in case of zapping */ -+ if (katom->kds_rset) -+ katom->kds_dep_satisfied = true; -+#endif /* CONFIG_KDS */ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ kbase_dma_fence_signal(katom); -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+ kbase_gpu_vm_lock(katom->kctx); -+ /* only roll back if extres is non-NULL */ -+ if (katom->extres) { -+ u32 res_no; -+ -+ res_no = katom->nr_extres; -+ while (res_no-- > 0) { -+ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; -+ struct kbase_va_region *reg; -+ -+ reg = kbase_region_tracker_find_region_base_address( -+ katom->kctx, -+ katom->extres[res_no].gpu_address); -+ kbase_unmap_external_resource(katom->kctx, reg, alloc); -+ } -+ kfree(katom->extres); -+ katom->extres = NULL; -+ } -+ kbase_gpu_vm_unlock(katom->kctx); -+} -+ -+/* -+ * Set up external resources needed by this job. -+ * -+ * jctx.lock must be held when this is called. -+ */ -+ -+static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) -+{ -+ int err_ret_val = -EINVAL; -+ u32 res_no; -+#ifdef CONFIG_KDS -+ u32 kds_res_count = 0; -+ struct kds_resource **kds_resources = NULL; -+ unsigned long *kds_access_bitmap = NULL; -+#endif /* CONFIG_KDS */ -+#ifdef CONFIG_MALI_DMA_FENCE -+ struct kbase_dma_fence_resv_info info = { -+ .dma_fence_resv_count = 0, -+ }; -+#ifdef CONFIG_SYNC -+ /* -+ * When both dma-buf fence and Android native sync is enabled, we -+ * disable dma-buf fence for contexts that are using Android native -+ * fences. -+ */ -+ const bool implicit_sync = !kbase_ctx_flag(katom->kctx, -+ KCTX_NO_IMPLICIT_SYNC); -+#else /* CONFIG_SYNC */ -+ const bool implicit_sync = true; -+#endif /* CONFIG_SYNC */ -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ struct base_external_resource *input_extres; -+ -+ KBASE_DEBUG_ASSERT(katom); -+ KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); -+ -+ /* no resources encoded, early out */ -+ if (!katom->nr_extres) -+ return -EINVAL; -+ -+ katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); -+ if (NULL == katom->extres) { -+ err_ret_val = -ENOMEM; -+ goto early_err_out; -+ } -+ -+ /* copy user buffer to the end of our real buffer. -+ * Make sure the struct sizes haven't changed in a way -+ * we don't support */ -+ BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); -+ input_extres = (struct base_external_resource *) -+ (((unsigned char *)katom->extres) + -+ (sizeof(*katom->extres) - sizeof(*input_extres)) * -+ katom->nr_extres); -+ -+ if (copy_from_user(input_extres, -+ get_compat_pointer(katom->kctx, &user_atom->extres_list), -+ sizeof(*input_extres) * katom->nr_extres) != 0) { -+ err_ret_val = -EINVAL; -+ goto early_err_out; -+ } -+#ifdef CONFIG_KDS -+ /* assume we have to wait for all */ -+ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); -+ kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL); -+ -+ if (!kds_resources) { -+ err_ret_val = -ENOMEM; -+ goto early_err_out; -+ } -+ -+ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); -+ kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), -+ sizeof(unsigned long), -+ GFP_KERNEL); -+ if (!kds_access_bitmap) { -+ err_ret_val = -ENOMEM; -+ goto early_err_out; -+ } -+#endif /* CONFIG_KDS */ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ if (implicit_sync) { -+ info.resv_objs = kmalloc_array(katom->nr_extres, -+ sizeof(struct reservation_object *), -+ GFP_KERNEL); -+ if (!info.resv_objs) { -+ err_ret_val = -ENOMEM; -+ goto early_err_out; -+ } -+ -+ info.dma_fence_excl_bitmap = -+ kcalloc(BITS_TO_LONGS(katom->nr_extres), -+ sizeof(unsigned long), GFP_KERNEL); -+ if (!info.dma_fence_excl_bitmap) { -+ err_ret_val = -ENOMEM; -+ goto early_err_out; -+ } -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+ /* Take the processes mmap lock */ -+ down_read(¤t->mm->mmap_lock); -+ -+ /* need to keep the GPU VM locked while we set up UMM buffers */ -+ kbase_gpu_vm_lock(katom->kctx); -+ for (res_no = 0; res_no < katom->nr_extres; res_no++) { -+ struct base_external_resource *res; -+ struct kbase_va_region *reg; -+ struct kbase_mem_phy_alloc *alloc; -+ bool exclusive; -+ -+ res = &input_extres[res_no]; -+ exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) -+ ? true : false; -+ reg = kbase_region_tracker_find_region_enclosing_address( -+ katom->kctx, -+ res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); -+ /* did we find a matching region object? */ -+ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) { -+ /* roll back */ -+ goto failed_loop; -+ } -+ -+ if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && -+ (reg->flags & KBASE_REG_SECURE)) { -+ katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; -+ } -+ -+ alloc = kbase_map_external_resource(katom->kctx, reg, -+ current->mm -+#ifdef CONFIG_KDS -+ , &kds_res_count, kds_resources, -+ kds_access_bitmap, exclusive -+#endif -+ ); -+ if (!alloc) { -+ err_ret_val = -EINVAL; -+ goto failed_loop; -+ } -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ if (implicit_sync && -+ reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { -+ struct reservation_object *resv; -+ -+ resv = reg->gpu_alloc->imported.umm.dma_buf->resv; -+ if (resv) -+ kbase_dma_fence_add_reservation(resv, &info, -+ exclusive); -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+ /* finish with updating out array with the data we found */ -+ /* NOTE: It is important that this is the last thing we do (or -+ * at least not before the first write) as we overwrite elements -+ * as we loop and could be overwriting ourself, so no writes -+ * until the last read for an element. -+ * */ -+ katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ -+ katom->extres[res_no].alloc = alloc; -+ } -+ /* successfully parsed the extres array */ -+ /* drop the vm lock before we call into kds */ -+ kbase_gpu_vm_unlock(katom->kctx); -+ -+ /* Release the processes mmap lock */ -+ up_read(¤t->mm->mmap_lock); -+ -+#ifdef CONFIG_KDS -+ if (kds_res_count) { -+ int wait_failed; -+ -+ /* We have resources to wait for with kds */ -+ katom->kds_dep_satisfied = false; -+ -+ wait_failed = kds_async_waitall(&katom->kds_rset, -+ &katom->kctx->jctx.kds_cb, katom, NULL, -+ kds_res_count, kds_access_bitmap, -+ kds_resources); -+ -+ if (wait_failed) -+ goto failed_kds_setup; -+ else -+ kbase_jd_kds_waiters_add(katom); -+ } else { -+ /* Nothing to wait for, so kds dep met */ -+ katom->kds_dep_satisfied = true; -+ } -+ kfree(kds_resources); -+ kfree(kds_access_bitmap); -+#endif /* CONFIG_KDS */ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ if (implicit_sync) { -+ if (info.dma_fence_resv_count) { -+ int ret; -+ -+ ret = kbase_dma_fence_wait(katom, &info); -+ if (ret < 0) -+ goto failed_dma_fence_setup; -+ } -+ -+ kfree(info.resv_objs); -+ kfree(info.dma_fence_excl_bitmap); -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+ /* all done OK */ -+ return 0; -+ -+/* error handling section */ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+failed_dma_fence_setup: -+#ifdef CONFIG_KDS -+ /* If we are here, dma_fence setup failed but KDS didn't. -+ * Revert KDS setup if any. -+ */ -+ if (kds_res_count) { -+ mutex_unlock(&katom->kctx->jctx.lock); -+ kds_resource_set_release_sync(&katom->kds_rset); -+ mutex_lock(&katom->kctx->jctx.lock); -+ -+ kbase_jd_kds_waiters_remove(katom); -+ katom->kds_dep_satisfied = true; -+ } -+#endif /* CONFIG_KDS */ -+#endif /* CONFIG_MALI_DMA_FENCE */ -+#ifdef CONFIG_KDS -+failed_kds_setup: -+#endif -+#if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) -+ /* Lock the processes mmap lock */ -+ down_read(¤t->mm->mmap_lock); -+ -+ /* lock before we unmap */ -+ kbase_gpu_vm_lock(katom->kctx); -+#endif -+ -+ failed_loop: -+ /* undo the loop work */ -+ while (res_no-- > 0) { -+ struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; -+ -+ kbase_unmap_external_resource(katom->kctx, NULL, alloc); -+ } -+ kbase_gpu_vm_unlock(katom->kctx); -+ -+ /* Release the processes mmap lock */ -+ up_read(¤t->mm->mmap_lock); -+ -+ early_err_out: -+ kfree(katom->extres); -+ katom->extres = NULL; -+#ifdef CONFIG_KDS -+ kfree(kds_resources); -+ kfree(kds_access_bitmap); -+#endif /* CONFIG_KDS */ -+#ifdef CONFIG_MALI_DMA_FENCE -+ if (implicit_sync) { -+ kfree(info.resv_objs); -+ kfree(info.dma_fence_excl_bitmap); -+ } -+#endif -+ return err_ret_val; -+} -+ -+static inline void jd_resolve_dep(struct list_head *out_list, -+ struct kbase_jd_atom *katom, -+ u8 d, bool ctx_is_dying) -+{ -+ u8 other_d = !d; -+ -+ while (!list_empty(&katom->dep_head[d])) { -+ struct kbase_jd_atom *dep_atom; -+ struct kbase_jd_atom *other_dep_atom; -+ u8 dep_type; -+ -+ dep_atom = list_entry(katom->dep_head[d].next, -+ struct kbase_jd_atom, dep_item[d]); -+ list_del(katom->dep_head[d].next); -+ -+ dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); -+ kbase_jd_katom_dep_clear(&dep_atom->dep[d]); -+ -+ if (katom->event_code != BASE_JD_EVENT_DONE && -+ (dep_type != BASE_JD_DEP_TYPE_ORDER)) { -+#ifdef CONFIG_KDS -+ if (!dep_atom->kds_dep_satisfied) { -+ /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and -+ * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up -+ */ -+ dep_atom->kds_dep_satisfied = true; -+ } -+#endif -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ kbase_dma_fence_cancel_callbacks(dep_atom); -+#endif -+ -+ dep_atom->event_code = katom->event_code; -+ KBASE_DEBUG_ASSERT(dep_atom->status != -+ KBASE_JD_ATOM_STATE_UNUSED); -+ -+ if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY) -+ != BASE_JD_REQ_SOFT_REPLAY) { -+ dep_atom->will_fail_event_code = -+ dep_atom->event_code; -+ } else { -+ dep_atom->status = -+ KBASE_JD_ATOM_STATE_COMPLETED; -+ } -+ } -+ other_dep_atom = (struct kbase_jd_atom *) -+ kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); -+ -+ if (!dep_atom->in_jd_list && (!other_dep_atom || -+ (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && -+ !dep_atom->will_fail_event_code && -+ !other_dep_atom->will_fail_event_code))) { -+ bool dep_satisfied = true; -+#ifdef CONFIG_MALI_DMA_FENCE -+ int dep_count; -+ -+ dep_count = kbase_fence_dep_count_read(dep_atom); -+ if (likely(dep_count == -1)) { -+ dep_satisfied = true; -+ } else { -+ /* -+ * There are either still active callbacks, or -+ * all fences for this @dep_atom has signaled, -+ * but the worker that will queue the atom has -+ * not yet run. -+ * -+ * Wait for the fences to signal and the fence -+ * worker to run and handle @dep_atom. If -+ * @dep_atom was completed due to error on -+ * @katom, then the fence worker will pick up -+ * the complete status and error code set on -+ * @dep_atom above. -+ */ -+ dep_satisfied = false; -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+#ifdef CONFIG_KDS -+ dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; -+#endif -+ -+ if (dep_satisfied) { -+ dep_atom->in_jd_list = true; -+ list_add_tail(&dep_atom->jd_item, out_list); -+ } -+ } -+ } -+} -+ -+KBASE_EXPORT_TEST_API(jd_resolve_dep); -+ -+#if MALI_CUSTOMER_RELEASE == 0 -+static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -+{ -+ kbdev->force_replay_count++; -+ -+ if (kbdev->force_replay_count >= kbdev->force_replay_limit) { -+ kbdev->force_replay_count = 0; -+ katom->event_code = BASE_JD_EVENT_FORCE_REPLAY; -+ -+ if (kbdev->force_replay_random) -+ kbdev->force_replay_limit = -+ (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1; -+ -+ dev_info(kbdev->dev, "force_replay : promoting to error\n"); -+ } -+} -+ -+/** Test to see if atom should be forced to fail. -+ * -+ * This function will check if an atom has a replay job as a dependent. If so -+ * then it will be considered for forced failure. */ -+static void jd_check_force_failure(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ int i; -+ -+ if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) || -+ (katom->core_req & BASEP_JD_REQ_EVENT_NEVER)) -+ return; -+ -+ for (i = 1; i < BASE_JD_ATOM_COUNT; i++) { -+ if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom || -+ kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) { -+ struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i]; -+ -+ if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == -+ BASE_JD_REQ_SOFT_REPLAY && -+ (dep_atom->core_req & kbdev->force_replay_core_req) -+ == kbdev->force_replay_core_req) { -+ jd_force_failure(kbdev, katom); -+ return; -+ } -+ } -+ } -+} -+#endif -+ -+/** -+ * is_dep_valid - Validate that a dependency is valid for early dependency -+ * submission -+ * @katom: Dependency atom to validate -+ * -+ * A dependency is valid if any of the following are true : -+ * - It does not exist (a non-existent dependency does not block submission) -+ * - It is in the job scheduler -+ * - It has completed, does not have a failure event code, and has not been -+ * marked to fail in the future -+ * -+ * Return: true if valid, false otherwise -+ */ -+static bool is_dep_valid(struct kbase_jd_atom *katom) -+{ -+ /* If there's no dependency then this is 'valid' from the perspective of -+ * early dependency submission */ -+ if (!katom) -+ return true; -+ -+ /* Dependency must have reached the job scheduler */ -+ if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) -+ return false; -+ -+ /* If dependency has completed and has failed or will fail then it is -+ * not valid */ -+ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && -+ (katom->event_code != BASE_JD_EVENT_DONE || -+ katom->will_fail_event_code)) -+ return false; -+ -+ return true; -+} -+ -+static void jd_try_submitting_deps(struct list_head *out_list, -+ struct kbase_jd_atom *node) -+{ -+ int i; -+ -+ for (i = 0; i < 2; i++) { -+ struct list_head *pos; -+ -+ list_for_each(pos, &node->dep_head[i]) { -+ struct kbase_jd_atom *dep_atom = list_entry(pos, -+ struct kbase_jd_atom, dep_item[i]); -+ -+ if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { -+ /*Check if atom deps look sane*/ -+ bool dep0_valid = is_dep_valid( -+ dep_atom->dep[0].atom); -+ bool dep1_valid = is_dep_valid( -+ dep_atom->dep[1].atom); -+ bool dep_satisfied = true; -+#ifdef CONFIG_MALI_DMA_FENCE -+ int dep_count; -+ -+ dep_count = kbase_fence_dep_count_read( -+ dep_atom); -+ if (likely(dep_count == -1)) { -+ dep_satisfied = true; -+ } else { -+ /* -+ * There are either still active callbacks, or -+ * all fences for this @dep_atom has signaled, -+ * but the worker that will queue the atom has -+ * not yet run. -+ * -+ * Wait for the fences to signal and the fence -+ * worker to run and handle @dep_atom. If -+ * @dep_atom was completed due to error on -+ * @katom, then the fence worker will pick up -+ * the complete status and error code set on -+ * @dep_atom above. -+ */ -+ dep_satisfied = false; -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+#ifdef CONFIG_KDS -+ dep_satisfied = dep_satisfied && -+ dep_atom->kds_dep_satisfied; -+#endif -+ -+ if (dep0_valid && dep1_valid && dep_satisfied) { -+ dep_atom->in_jd_list = true; -+ list_add(&dep_atom->jd_item, out_list); -+ } -+ } -+ } -+ } -+} -+ -+/* -+ * Perform the necessary handling of an atom that has finished running -+ * on the GPU. -+ * -+ * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller -+ * is responsible for calling kbase_finish_soft_job *before* calling this function. -+ * -+ * The caller must hold the kbase_jd_context.lock. -+ */ -+bool jd_done_nolock(struct kbase_jd_atom *katom, -+ struct list_head *completed_jobs_ctx) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct list_head completed_jobs; -+ struct list_head runnable_jobs; -+ bool need_to_try_schedule_context = false; -+ int i; -+ -+ INIT_LIST_HEAD(&completed_jobs); -+ INIT_LIST_HEAD(&runnable_jobs); -+ -+ KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); -+ -+#if MALI_CUSTOMER_RELEASE == 0 -+ jd_check_force_failure(katom); -+#endif -+ -+ /* This is needed in case an atom is failed due to being invalid, this -+ * can happen *before* the jobs that the atom depends on have completed */ -+ for (i = 0; i < 2; i++) { -+ if (kbase_jd_katom_dep_atom(&katom->dep[i])) { -+ list_del(&katom->dep_item[i]); -+ kbase_jd_katom_dep_clear(&katom->dep[i]); -+ } -+ } -+ -+ /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with -+ * BASE_JD_EVENT_TILE_RANGE_FAULT. -+ * -+ * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the -+ * error code to BASE_JD_EVENT_DONE -+ */ -+ -+ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && -+ katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { -+ if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { -+ /* Promote the failure to job done */ -+ katom->event_code = BASE_JD_EVENT_DONE; -+ katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); -+ } -+ } -+ -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ list_add_tail(&katom->jd_item, &completed_jobs); -+ -+ while (!list_empty(&completed_jobs)) { -+ katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); -+ list_del(completed_jobs.prev); -+ KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); -+ -+ for (i = 0; i < 2; i++) -+ jd_resolve_dep(&runnable_jobs, katom, i, -+ kbase_ctx_flag(kctx, KCTX_DYING)); -+ -+ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) -+ kbase_jd_post_external_resources(katom); -+ -+ while (!list_empty(&runnable_jobs)) { -+ struct kbase_jd_atom *node; -+ -+ node = list_entry(runnable_jobs.next, -+ struct kbase_jd_atom, jd_item); -+ list_del(runnable_jobs.next); -+ node->in_jd_list = false; -+ -+ KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); -+ -+ if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && -+ !kbase_ctx_flag(kctx, KCTX_DYING)) { -+ need_to_try_schedule_context |= jd_run_atom(node); -+ } else { -+ node->event_code = katom->event_code; -+ -+ if ((node->core_req & -+ BASE_JD_REQ_SOFT_JOB_TYPE) == -+ BASE_JD_REQ_SOFT_REPLAY) { -+ if (kbase_replay_process(node)) -+ /* Don't complete this atom */ -+ continue; -+ } else if (node->core_req & -+ BASE_JD_REQ_SOFT_JOB) { -+ /* If this is a fence wait soft job -+ * then remove it from the list of sync -+ * waiters. -+ */ -+ if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req) -+ kbasep_remove_waiting_soft_job(node); -+ -+ kbase_finish_soft_job(node); -+ } -+ node->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ } -+ -+ if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { -+ list_add_tail(&node->jd_item, &completed_jobs); -+ } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && -+ !node->will_fail_event_code) { -+ /* Node successfully submitted, try submitting -+ * dependencies as they may now be representable -+ * in JS */ -+ jd_try_submitting_deps(&runnable_jobs, node); -+ } -+ } -+ -+ /* Register a completed job as a disjoint event when the GPU -+ * is in a disjoint state (ie. being reset or replaying jobs). -+ */ -+ kbase_disjoint_event_potential(kctx->kbdev); -+ if (completed_jobs_ctx) -+ list_add_tail(&katom->jd_item, completed_jobs_ctx); -+ else -+ kbase_event_post(kctx, katom); -+ -+ /* Decrement and check the TOTAL number of jobs. This includes -+ * those not tracked by the scheduler: 'not ready to run' and -+ * 'dependency-only' jobs. */ -+ if (--kctx->jctx.job_nr == 0) -+ wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter -+ * that we've got no more jobs (so we can be safely terminated) */ -+ } -+ -+ return need_to_try_schedule_context; -+} -+ -+KBASE_EXPORT_TEST_API(jd_done_nolock); -+ -+#ifdef CONFIG_GPU_TRACEPOINTS -+enum { -+ CORE_REQ_DEP_ONLY, -+ CORE_REQ_SOFT, -+ CORE_REQ_COMPUTE, -+ CORE_REQ_FRAGMENT, -+ CORE_REQ_VERTEX, -+ CORE_REQ_TILER, -+ CORE_REQ_FRAGMENT_VERTEX, -+ CORE_REQ_FRAGMENT_VERTEX_TILER, -+ CORE_REQ_FRAGMENT_TILER, -+ CORE_REQ_VERTEX_TILER, -+ CORE_REQ_UNKNOWN -+}; -+static const char * const core_req_strings[] = { -+ "Dependency Only Job", -+ "Soft Job", -+ "Compute Shader Job", -+ "Fragment Shader Job", -+ "Vertex/Geometry Shader Job", -+ "Tiler Job", -+ "Fragment Shader + Vertex/Geometry Shader Job", -+ "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", -+ "Fragment Shader + Tiler Job", -+ "Vertex/Geometry Shader Job + Tiler Job", -+ "Unknown Job" -+}; -+static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) -+{ -+ if (core_req & BASE_JD_REQ_SOFT_JOB) -+ return core_req_strings[CORE_REQ_SOFT]; -+ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) -+ return core_req_strings[CORE_REQ_COMPUTE]; -+ switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { -+ case BASE_JD_REQ_DEP: -+ return core_req_strings[CORE_REQ_DEP_ONLY]; -+ case BASE_JD_REQ_FS: -+ return core_req_strings[CORE_REQ_FRAGMENT]; -+ case BASE_JD_REQ_CS: -+ return core_req_strings[CORE_REQ_VERTEX]; -+ case BASE_JD_REQ_T: -+ return core_req_strings[CORE_REQ_TILER]; -+ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): -+ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; -+ case (BASE_JD_REQ_FS | BASE_JD_REQ_T): -+ return core_req_strings[CORE_REQ_FRAGMENT_TILER]; -+ case (BASE_JD_REQ_CS | BASE_JD_REQ_T): -+ return core_req_strings[CORE_REQ_VERTEX_TILER]; -+ case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): -+ return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; -+ } -+ return core_req_strings[CORE_REQ_UNKNOWN]; -+} -+#endif -+ -+bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom) -+{ -+ struct kbase_jd_context *jctx = &kctx->jctx; -+ int queued = 0; -+ int i; -+ int sched_prio; -+ bool ret; -+ bool will_fail = false; -+ -+ /* Update the TOTAL number of jobs. This includes those not tracked by -+ * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ -+ jctx->job_nr++; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ katom->start_timestamp.tv64 = 0; -+#else -+ katom->start_timestamp = 0; -+#endif -+ katom->udata = user_atom->udata; -+ katom->kctx = kctx; -+ katom->nr_extres = user_atom->nr_extres; -+ katom->extres = NULL; -+ katom->device_nr = user_atom->device_nr; -+ katom->affinity = 0; -+ katom->jc = user_atom->jc; -+ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; -+ katom->core_req = user_atom->core_req; -+ katom->atom_flags = 0; -+ katom->retry_count = 0; -+ katom->need_cache_flush_cores_retained = 0; -+ katom->pre_dep = NULL; -+ katom->post_dep = NULL; -+ katom->x_pre_dep = NULL; -+ katom->x_post_dep = NULL; -+ katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; -+ -+ /* Implicitly sets katom->protected_state.enter as well. */ -+ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; -+ -+ katom->age = kctx->age_count++; -+ -+ INIT_LIST_HEAD(&katom->jd_item); -+#ifdef CONFIG_KDS -+ /* Start by assuming that the KDS dependencies are satisfied, -+ * kbase_jd_pre_external_resources will correct this if there are dependencies */ -+ katom->kds_dep_satisfied = true; -+ katom->kds_rset = NULL; -+#endif /* CONFIG_KDS */ -+#ifdef CONFIG_MALI_DMA_FENCE -+ kbase_fence_dep_count_set(katom, -1); -+#endif -+ -+ /* Don't do anything if there is a mess up with dependencies. -+ This is done in a separate cycle to check both the dependencies at ones, otherwise -+ it will be extra complexity to deal with 1st dependency ( just added to the list ) -+ if only the 2nd one has invalid config. -+ */ -+ for (i = 0; i < 2; i++) { -+ int dep_atom_number = user_atom->pre_dep[i].atom_id; -+ base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; -+ -+ if (dep_atom_number) { -+ if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && -+ dep_atom_type != BASE_JD_DEP_TYPE_DATA) { -+ katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ -+ /* Wrong dependency setup. Atom will be sent -+ * back to user space. Do not record any -+ * dependencies. */ -+ KBASE_TLSTREAM_TL_NEW_ATOM( -+ katom, -+ kbase_jd_atom_id(kctx, katom)); -+ KBASE_TLSTREAM_TL_RET_ATOM_CTX( -+ katom, kctx); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, -+ TL_ATOM_STATE_IDLE); -+ -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ } -+ } -+ -+ /* Add dependencies */ -+ for (i = 0; i < 2; i++) { -+ int dep_atom_number = user_atom->pre_dep[i].atom_id; -+ base_jd_dep_type dep_atom_type; -+ struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; -+ -+ dep_atom_type = user_atom->pre_dep[i].dependency_type; -+ kbase_jd_katom_dep_clear(&katom->dep[i]); -+ -+ if (!dep_atom_number) -+ continue; -+ -+ if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || -+ dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { -+ -+ if (dep_atom->event_code == BASE_JD_EVENT_DONE) -+ continue; -+ /* don't stop this atom if it has an order dependency -+ * only to the failed one, try to submit it through -+ * the normal path -+ */ -+ if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && -+ dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { -+ continue; -+ } -+ -+ /* Atom has completed, propagate the error code if any */ -+ katom->event_code = dep_atom->event_code; -+ katom->status = KBASE_JD_ATOM_STATE_QUEUED; -+ -+ /* This atom is going through soft replay or -+ * will be sent back to user space. Do not record any -+ * dependencies. */ -+ KBASE_TLSTREAM_TL_NEW_ATOM( -+ katom, -+ kbase_jd_atom_id(kctx, katom)); -+ KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, -+ TL_ATOM_STATE_IDLE); -+ -+ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) -+ == BASE_JD_REQ_SOFT_REPLAY) { -+ if (kbase_replay_process(katom)) { -+ ret = false; -+ goto out; -+ } -+ } -+ will_fail = true; -+ -+ } else { -+ /* Atom is in progress, add this atom to the list */ -+ list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); -+ kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); -+ queued = 1; -+ } -+ } -+ -+ if (will_fail) { -+ if (!queued) { -+ ret = jd_done_nolock(katom, NULL); -+ -+ goto out; -+ } else { -+ katom->will_fail_event_code = katom->event_code; -+ ret = false; -+ -+ goto out; -+ } -+ } else { -+ /* These must occur after the above loop to ensure that an atom -+ * that depends on a previous atom with the same number behaves -+ * as expected */ -+ katom->event_code = BASE_JD_EVENT_DONE; -+ katom->status = KBASE_JD_ATOM_STATE_QUEUED; -+ } -+ -+ /* For invalid priority, be most lenient and choose the default */ -+ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); -+ if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) -+ sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; -+ katom->sched_priority = sched_prio; -+ -+ /* Create a new atom recording all dependencies it was set up with. */ -+ KBASE_TLSTREAM_TL_NEW_ATOM( -+ katom, -+ kbase_jd_atom_id(kctx, katom)); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority); -+ KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); -+ for (i = 0; i < 2; i++) -+ if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type( -+ &katom->dep[i])) { -+ KBASE_TLSTREAM_TL_DEP_ATOM_ATOM( -+ (void *)kbase_jd_katom_dep_atom( -+ &katom->dep[i]), -+ (void *)katom); -+ } else if (BASE_JD_DEP_TYPE_INVALID != -+ user_atom->pre_dep[i].dependency_type) { -+ /* Resolved dependency. */ -+ int dep_atom_number = -+ user_atom->pre_dep[i].atom_id; -+ struct kbase_jd_atom *dep_atom = -+ &jctx->atoms[dep_atom_number]; -+ -+ KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM( -+ (void *)dep_atom, -+ (void *)katom); -+ } -+ -+ /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */ -+ if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { -+ dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL"); -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ -+ /* Reject atoms with an invalid device_nr */ -+ if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && -+ (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { -+ dev_warn(kctx->kbdev->dev, -+ "Rejecting atom with invalid device_nr %d", -+ katom->device_nr); -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ -+ /* Reject atoms with invalid core requirements */ -+ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && -+ (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { -+ dev_warn(kctx->kbdev->dev, -+ "Rejecting atom with invalid core requirements"); -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ -+ if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { -+ /* handle what we need to do to access the external resources */ -+ if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { -+ /* setup failed (no access, bad resource, unknown resource types, etc.) */ -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ } -+ -+ /* Validate the atom. Function will return error if the atom is -+ * malformed. -+ * -+ * Soft-jobs never enter the job scheduler but have their own initialize method. -+ * -+ * If either fail then we immediately complete the atom with an error. -+ */ -+ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { -+ if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ } else { -+ /* Soft-job */ -+ if (kbase_prepare_soft_job(katom) != 0) { -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ } -+ -+#ifdef CONFIG_GPU_TRACEPOINTS -+ katom->work_id = atomic_inc_return(&jctx->work_id); -+ trace_gpu_job_enqueue((u32)kctx->id, katom->work_id, -+ kbasep_map_core_reqs_to_string(katom->core_req)); -+#endif -+ -+ if (queued && !IS_GPU_ATOM(katom)) { -+ ret = false; -+ goto out; -+ } -+#ifdef CONFIG_KDS -+ if (!katom->kds_dep_satisfied) { -+ /* Queue atom due to KDS dependency */ -+ ret = false; -+ goto out; -+ } -+#endif /* CONFIG_KDS */ -+ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ if (kbase_fence_dep_count_read(katom) != -1) { -+ ret = false; -+ goto out; -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) -+ == BASE_JD_REQ_SOFT_REPLAY) { -+ if (kbase_replay_process(katom)) -+ ret = false; -+ else -+ ret = jd_done_nolock(katom, NULL); -+ -+ goto out; -+ } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { -+ if (kbase_process_soft_job(katom) == 0) { -+ kbase_finish_soft_job(katom); -+ ret = jd_done_nolock(katom, NULL); -+ goto out; -+ } -+ -+ ret = false; -+ } else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { -+ katom->status = KBASE_JD_ATOM_STATE_IN_JS; -+ ret = kbasep_js_add_job(kctx, katom); -+ /* If job was cancelled then resolve immediately */ -+ if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED) -+ ret = jd_done_nolock(katom, NULL); -+ } else { -+ /* This is a pure dependency. Resolve it immediately */ -+ ret = jd_done_nolock(katom, NULL); -+ } -+ -+ out: -+ return ret; -+} -+ -+int kbase_jd_submit(struct kbase_context *kctx, -+ void __user *user_addr, u32 nr_atoms, u32 stride, -+ bool uk6_atom) -+{ -+ struct kbase_jd_context *jctx = &kctx->jctx; -+ int err = 0; -+ int i; -+ bool need_to_try_schedule_context = false; -+ struct kbase_device *kbdev; -+ u32 latest_flush; -+ -+ /* -+ * kbase_jd_submit isn't expected to fail and so all errors with the -+ * jobs are reported by immediately failing them (through event system) -+ */ -+ kbdev = kctx->kbdev; -+ -+ beenthere(kctx, "%s", "Enter"); -+ -+ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { -+ dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it"); -+ return -EINVAL; -+ } -+ -+ if (stride != sizeof(base_jd_atom_v2)) { -+ dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); -+ return -EINVAL; -+ } -+ -+ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, -+ &kctx->timeline.jd_atoms_in_flight)); -+ -+ /* All atoms submitted in this call have the same flush ID */ -+ latest_flush = kbase_backend_get_current_flush_id(kbdev); -+ -+ for (i = 0; i < nr_atoms; i++) { -+ struct base_jd_atom_v2 user_atom; -+ struct kbase_jd_atom *katom; -+ -+#ifdef BASE_LEGACY_UK6_SUPPORT -+ BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) != -+ sizeof(base_jd_atom_v2)); -+ -+ if (uk6_atom) { -+ struct base_jd_atom_v2_uk6 user_atom_v6; -+ base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA}; -+ -+ if (copy_from_user(&user_atom_v6, user_addr, -+ sizeof(user_atom_v6))) { -+ err = -EINVAL; -+ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, -+ atomic_sub_return( -+ nr_atoms - i, -+ &kctx->timeline.jd_atoms_in_flight)); -+ break; -+ } -+ /* Convert from UK6 atom format to UK7 format */ -+ user_atom.jc = user_atom_v6.jc; -+ user_atom.udata = user_atom_v6.udata; -+ user_atom.extres_list = user_atom_v6.extres_list; -+ user_atom.nr_extres = user_atom_v6.nr_extres; -+ user_atom.core_req = (u32)(user_atom_v6.core_req & 0x7fff); -+ -+ /* atom number 0 is used for no dependency atoms */ -+ if (!user_atom_v6.pre_dep[0]) -+ dep_types[0] = BASE_JD_DEP_TYPE_INVALID; -+ -+ base_jd_atom_dep_set(&user_atom.pre_dep[0], -+ user_atom_v6.pre_dep[0], -+ dep_types[0]); -+ -+ /* atom number 0 is used for no dependency atoms */ -+ if (!user_atom_v6.pre_dep[1]) -+ dep_types[1] = BASE_JD_DEP_TYPE_INVALID; -+ -+ base_jd_atom_dep_set(&user_atom.pre_dep[1], -+ user_atom_v6.pre_dep[1], -+ dep_types[1]); -+ -+ user_atom.atom_number = user_atom_v6.atom_number; -+ user_atom.prio = user_atom_v6.prio; -+ user_atom.device_nr = user_atom_v6.device_nr; -+ } else { -+#endif /* BASE_LEGACY_UK6_SUPPORT */ -+ if (copy_from_user(&user_atom, user_addr, -+ sizeof(user_atom)) != 0) { -+ err = -EINVAL; -+ KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, -+ atomic_sub_return(nr_atoms - i, -+ &kctx->timeline.jd_atoms_in_flight)); -+ break; -+ } -+#ifdef BASE_LEGACY_UK6_SUPPORT -+ } -+#endif -+ -+#ifdef BASE_LEGACY_UK10_2_SUPPORT -+ if (KBASE_API_VERSION(10, 3) > kctx->api_version) -+ user_atom.core_req = (u32)(user_atom.compat_core_req -+ & 0x7fff); -+#endif /* BASE_LEGACY_UK10_2_SUPPORT */ -+ -+ user_addr = (void __user *)((uintptr_t) user_addr + stride); -+ -+ mutex_lock(&jctx->lock); -+#ifndef compiletime_assert -+#define compiletime_assert_defined -+#define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ -+while (false) -+#endif -+ compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) >= -+ BASE_JD_ATOM_COUNT, -+ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); -+ compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == -+ sizeof(user_atom.atom_number), -+ "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); -+#ifdef compiletime_assert_defined -+#undef compiletime_assert -+#undef compiletime_assert_defined -+#endif -+ if (user_atom.atom_number >= BASE_JD_ATOM_COUNT) { -+ err = -EINVAL; -+ break; -+ } -+ user_atom.atom_number = -+ array_index_nospec(user_atom.atom_number, -+ BASE_JD_ATOM_COUNT); -+ katom = &jctx->atoms[user_atom.atom_number]; -+ -+ /* Record the flush ID for the cache flush optimisation */ -+ katom->flush_id = latest_flush; -+ -+ while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { -+ /* Atom number is already in use, wait for the atom to -+ * complete -+ */ -+ mutex_unlock(&jctx->lock); -+ -+ /* This thread will wait for the atom to complete. Due -+ * to thread scheduling we are not sure that the other -+ * thread that owns the atom will also schedule the -+ * context, so we force the scheduler to be active and -+ * hence eventually schedule this context at some point -+ * later. -+ */ -+ kbase_js_sched_all(kbdev); -+ -+ if (wait_event_killable(katom->completed, -+ katom->status == -+ KBASE_JD_ATOM_STATE_UNUSED) != 0) { -+ /* We're being killed so the result code -+ * doesn't really matter -+ */ -+ return 0; -+ } -+ mutex_lock(&jctx->lock); -+ } -+ -+ need_to_try_schedule_context |= -+ jd_submit_atom(kctx, &user_atom, katom); -+ -+ /* Register a completed job as a disjoint event when the GPU is in a disjoint state -+ * (ie. being reset or replaying jobs). -+ */ -+ kbase_disjoint_event_potential(kbdev); -+ -+ mutex_unlock(&jctx->lock); -+ } -+ -+ if (need_to_try_schedule_context) -+ kbase_js_sched_all(kbdev); -+ -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_jd_submit); -+ -+void kbase_jd_done_worker(struct work_struct *data) -+{ -+ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); -+ struct kbase_jd_context *jctx; -+ struct kbase_context *kctx; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbase_device *kbdev; -+ struct kbasep_js_device_data *js_devdata; -+ u64 cache_jc = katom->jc; -+ struct kbasep_js_atom_retained_state katom_retained_state; -+ bool context_idle; -+ base_jd_core_req core_req = katom->core_req; -+ u64 affinity = katom->affinity; -+ enum kbase_atom_coreref_state coreref_state = katom->coreref_state; -+ -+ /* Soft jobs should never reach this function */ -+ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); -+ -+ kctx = katom->kctx; -+ jctx = &kctx->jctx; -+ kbdev = kctx->kbdev; -+ js_kctx_info = &kctx->jctx.sched_info; -+ js_devdata = &kbdev->js_data; -+ -+ KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); -+ -+ kbase_backend_complete_wq(kbdev, katom); -+ -+ /* -+ * Begin transaction on JD context and JS context -+ */ -+ mutex_lock(&jctx->lock); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE); -+ mutex_lock(&js_devdata->queue_mutex); -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ /* This worker only gets called on contexts that are scheduled *in*. This is -+ * because it only happens in response to an IRQ from a job that was -+ * running. -+ */ -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ if (katom->event_code == BASE_JD_EVENT_STOPPED) { -+ /* Atom has been promoted to stopped */ -+ unsigned long flags; -+ -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ katom->status = KBASE_JD_ATOM_STATE_IN_JS; -+ kbase_js_unpull(kctx, katom); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&jctx->lock); -+ -+ return; -+ } -+ -+ if (katom->event_code != BASE_JD_EVENT_DONE) -+ dev_err(kbdev->dev, -+ "t6xx: GPU fault 0x%02lx from job slot %d\n", -+ (unsigned long)katom->event_code, -+ katom->slot_nr); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) -+ kbase_as_poking_timer_release_atom(kbdev, kctx, katom); -+ -+ /* Retain state before the katom disappears */ -+ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); -+ -+ context_idle = kbase_js_complete_atom_wq(kctx, katom); -+ -+ KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); -+ -+ kbasep_js_remove_job(kbdev, kctx, katom); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; -+ /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ -+ jd_done_nolock(katom, &kctx->completed_jobs); -+ -+ /* katom may have been freed now, do not use! */ -+ -+ if (context_idle) { -+ unsigned long flags; -+ -+ context_idle = false; -+ mutex_lock(&js_devdata->queue_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* If kbase_sched() has scheduled this context back in then -+ * KCTX_ACTIVE will have been set after we marked it as -+ * inactive, and another pm reference will have been taken, so -+ * drop our reference. But do not call kbase_jm_idle_ctx(), as -+ * the context is active and fast-starting is allowed. -+ * -+ * If an atom has been fast-started then kctx->atoms_pulled will -+ * be non-zero but KCTX_ACTIVE will still be false (as the -+ * previous pm reference has been inherited). Do NOT drop our -+ * reference, as it has been re-used, and leave the context as -+ * active. -+ * -+ * If no new atoms have been started then KCTX_ACTIVE will still -+ * be false and atoms_pulled will be zero, so drop the reference -+ * and call kbase_jm_idle_ctx(). -+ * -+ * As the checks are done under both the queue_mutex and -+ * hwaccess_lock is should be impossible for this to race -+ * with the scheduler code. -+ */ -+ if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || -+ !atomic_read(&kctx->atoms_pulled)) { -+ /* Calling kbase_jm_idle_ctx() here will ensure that -+ * atoms are not fast-started when we drop the -+ * hwaccess_lock. This is not performed if -+ * KCTX_ACTIVE is set as in that case another pm -+ * reference has been taken and a fast-start would be -+ * valid. -+ */ -+ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) -+ kbase_jm_idle_ctx(kbdev, kctx); -+ context_idle = true; -+ } else { -+ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&js_devdata->queue_mutex); -+ } -+ -+ /* -+ * Transaction complete -+ */ -+ mutex_unlock(&jctx->lock); -+ -+ /* Job is now no longer running, so can now safely release the context -+ * reference, and handle any actions that were logged against the atom's retained state */ -+ -+ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); -+ -+ kbase_js_sched_all(kbdev); -+ -+ if (!atomic_dec_return(&kctx->work_count)) { -+ /* If worker now idle then post all events that jd_done_nolock() -+ * has queued */ -+ mutex_lock(&jctx->lock); -+ while (!list_empty(&kctx->completed_jobs)) { -+ struct kbase_jd_atom *atom = list_entry( -+ kctx->completed_jobs.next, -+ struct kbase_jd_atom, jd_item); -+ list_del(kctx->completed_jobs.next); -+ -+ kbase_event_post(kctx, atom); -+ } -+ mutex_unlock(&jctx->lock); -+ } -+ -+ kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, -+ coreref_state); -+ -+ if (context_idle) -+ kbase_pm_context_idle(kbdev); -+ -+ KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); -+} -+ -+/** -+ * jd_cancel_worker - Work queue job cancel function. -+ * @data: a &struct work_struct -+ * -+ * Only called as part of 'Zapping' a context (which occurs on termination). -+ * Operates serially with the kbase_jd_done_worker() on the work queue. -+ * -+ * This can only be called on contexts that aren't scheduled. -+ * -+ * We don't need to release most of the resources that would occur on -+ * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be -+ * running (by virtue of only being called on contexts that aren't -+ * scheduled). -+ */ -+static void jd_cancel_worker(struct work_struct *data) -+{ -+ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); -+ struct kbase_jd_context *jctx; -+ struct kbase_context *kctx; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ bool need_to_try_schedule_context; -+ bool attr_state_changed; -+ struct kbase_device *kbdev; -+ -+ /* Soft jobs should never reach this function */ -+ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); -+ -+ kctx = katom->kctx; -+ kbdev = kctx->kbdev; -+ jctx = &kctx->jctx; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); -+ -+ /* This only gets called on contexts that are scheduled out. Hence, we must -+ * make sure we don't de-ref the number of running jobs (there aren't -+ * any), nor must we try to schedule out the context (it's already -+ * scheduled out). -+ */ -+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ /* Scheduler: Remove the job from the system */ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ mutex_lock(&jctx->lock); -+ -+ need_to_try_schedule_context = jd_done_nolock(katom, NULL); -+ /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to -+ * schedule the context. There's also no need for the jsctx_mutex to have been taken -+ * around this too. */ -+ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); -+ -+ /* katom may have been freed now, do not use! */ -+ mutex_unlock(&jctx->lock); -+ -+ if (attr_state_changed) -+ kbase_js_sched_all(kbdev); -+} -+ -+/** -+ * kbase_jd_done - Complete a job that has been removed from the Hardware -+ * @katom: atom which has been completed -+ * @slot_nr: slot the atom was on -+ * @end_timestamp: completion time -+ * @done_code: completion code -+ * -+ * This must be used whenever a job has been removed from the Hardware, e.g.: -+ * An IRQ indicates that the job finished (for both error and 'done' codes), or -+ * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. -+ * -+ * Some work is carried out immediately, and the rest is deferred onto a -+ * workqueue -+ * -+ * Context: -+ * This can be called safely from atomic context. -+ * The caller must hold kbdev->hwaccess_lock -+ */ -+void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, -+ ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) -+{ -+ struct kbase_context *kctx; -+ struct kbase_device *kbdev; -+ -+ KBASE_DEBUG_ASSERT(katom); -+ kctx = katom->kctx; -+ KBASE_DEBUG_ASSERT(kctx); -+ kbdev = kctx->kbdev; -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) -+ katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; -+ -+ KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); -+ -+ kbase_job_check_leave_disjoint(kbdev, katom); -+ -+ katom->slot_nr = slot_nr; -+ -+ atomic_inc(&kctx->work_count); -+ -+#ifdef CONFIG_DEBUG_FS -+ /* a failed job happened and is waiting for dumping*/ -+ if (!katom->will_fail_event_code && -+ kbase_debug_job_fault_process(katom, katom->event_code)) -+ return; -+#endif -+ -+ WARN_ON(work_pending(&katom->work)); -+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); -+ INIT_WORK(&katom->work, kbase_jd_done_worker); -+ queue_work(kctx->jctx.job_done_wq, &katom->work); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_jd_done); -+ -+void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx; -+ -+ KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(NULL != katom); -+ kctx = katom->kctx; -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); -+ -+ /* This should only be done from a context that is not scheduled */ -+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ WARN_ON(work_pending(&katom->work)); -+ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); -+ INIT_WORK(&katom->work, jd_cancel_worker); -+ queue_work(kctx->jctx.job_done_wq, &katom->work); -+} -+ -+ -+void kbase_jd_zap_context(struct kbase_context *kctx) -+{ -+ struct kbase_jd_atom *katom; -+ struct list_head *entry, *tmp; -+ struct kbase_device *kbdev; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ kbdev = kctx->kbdev; -+ -+ KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); -+ -+ kbase_js_zap_context(kctx); -+ -+ mutex_lock(&kctx->jctx.lock); -+ -+ /* -+ * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are -+ * queued outside the job scheduler. -+ */ -+ -+ del_timer_sync(&kctx->soft_job_timeout); -+ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { -+ katom = list_entry(entry, struct kbase_jd_atom, queue); -+ kbase_cancel_soft_job(katom); -+ } -+ -+ -+#ifdef CONFIG_KDS -+ -+ /* For each job waiting on a kds resource, cancel the wait and force the job to -+ * complete early, this is done so that we don't leave jobs outstanding waiting -+ * on kds resources which may never be released when contexts are zapped, resulting -+ * in a hang. -+ * -+ * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held, -+ * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job. -+ */ -+ -+ list_for_each(entry, &kctx->waiting_kds_resource) { -+ katom = list_entry(entry, struct kbase_jd_atom, node); -+ -+ kbase_cancel_kds_wait_job(katom); -+ } -+#endif -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ kbase_dma_fence_cancel_all_atoms(kctx); -+#endif -+ -+ mutex_unlock(&kctx->jctx.lock); -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ /* Flush dma-fence workqueue to ensure that any callbacks that may have -+ * been queued are done before continuing. -+ */ -+ flush_workqueue(kctx->dma_fence.wq); -+#endif -+ -+ kbase_jm_wait_for_zero_jobs(kctx); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_jd_zap_context); -+ -+int kbase_jd_init(struct kbase_context *kctx) -+{ -+ int i; -+ int mali_err = 0; -+#ifdef CONFIG_KDS -+ int err; -+#endif /* CONFIG_KDS */ -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", -+ WQ_HIGHPRI | WQ_UNBOUND, 1); -+ if (NULL == kctx->jctx.job_done_wq) { -+ mali_err = -ENOMEM; -+ goto out1; -+ } -+ -+ for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { -+ init_waitqueue_head(&kctx->jctx.atoms[i].completed); -+ -+ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); -+ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); -+ -+ /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ -+ kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; -+ kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; -+ -+#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) -+ kctx->jctx.atoms[i].dma_fence.context = -+ dma_fence_context_alloc(1); -+ atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); -+ INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); -+#endif -+ } -+ -+ mutex_init(&kctx->jctx.lock); -+ -+ init_waitqueue_head(&kctx->jctx.zero_jobs_wait); -+ -+ spin_lock_init(&kctx->jctx.tb_lock); -+ -+#ifdef CONFIG_KDS -+ err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear); -+ if (0 != err) { -+ mali_err = -EINVAL; -+ goto out2; -+ } -+#endif /* CONFIG_KDS */ -+ -+ kctx->jctx.job_nr = 0; -+ INIT_LIST_HEAD(&kctx->completed_jobs); -+ atomic_set(&kctx->work_count, 0); -+ -+ return 0; -+ -+#ifdef CONFIG_KDS -+ out2: -+ destroy_workqueue(kctx->jctx.job_done_wq); -+#endif /* CONFIG_KDS */ -+ out1: -+ return mali_err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_jd_init); -+ -+void kbase_jd_exit(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+#ifdef CONFIG_KDS -+ kds_callback_term(&kctx->jctx.kds_cb); -+#endif /* CONFIG_KDS */ -+ /* Work queue is emptied by this */ -+ destroy_workqueue(kctx->jctx.job_done_wq); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_jd_exit); -diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c -new file mode 100755 -index 000000000..44643abf8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c -@@ -0,0 +1,233 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifdef CONFIG_DEBUG_FS -+ -+#include -+#include -+#include -+#include -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#include -+#endif -+ -+struct kbase_jd_debugfs_depinfo { -+ u8 id; -+ char type; -+}; -+ -+static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, -+ struct seq_file *sfile) -+{ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ struct kbase_sync_fence_info info; -+ int res; -+ -+ switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: -+ res = kbase_sync_fence_out_info_get(atom, &info); -+ if (res == 0) -+ seq_printf(sfile, "Sa([%p]%d) ", -+ info.fence, info.status); -+ break; -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ res = kbase_sync_fence_in_info_get(atom, &info); -+ if (res == 0) -+ seq_printf(sfile, "Wa([%p]%d) ", -+ info.fence, info.status); -+ break; -+ default: -+ break; -+ } -+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { -+ struct kbase_fence_cb *cb; -+ -+ if (atom->dma_fence.fence) { -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence = atom->dma_fence.fence; -+#else -+ struct dma_fence *fence = atom->dma_fence.fence; -+#endif -+ -+ seq_printf(sfile, -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+ "Sd(%u#%u: %s) ", -+#else -+ "Sd(%llu#%u: %s) ", -+#endif -+ fence->context, -+ fence->seqno, -+ dma_fence_is_signaled(fence) ? -+ "signaled" : "active"); -+ } -+ -+ list_for_each_entry(cb, &atom->dma_fence.callbacks, -+ node) { -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence = cb->fence; -+#else -+ struct dma_fence *fence = cb->fence; -+#endif -+ -+ seq_printf(sfile, -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+ "Wd(%u#%u: %s) ", -+#else -+ "Wd(%llu#%u: %s) ", -+#endif -+ fence->context, -+ fence->seqno, -+ dma_fence_is_signaled(fence) ? -+ "signaled" : "active"); -+ } -+ } -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ -+} -+ -+static void kbasep_jd_debugfs_atom_deps( -+ struct kbase_jd_debugfs_depinfo *deps, -+ struct kbase_jd_atom *atom) -+{ -+ struct kbase_context *kctx = atom->kctx; -+ int i; -+ -+ for (i = 0; i < 2; i++) { -+ deps[i].id = (unsigned)(atom->dep[i].atom ? -+ kbase_jd_atom_id(kctx, atom->dep[i].atom) : 0); -+ -+ switch (atom->dep[i].dep_type) { -+ case BASE_JD_DEP_TYPE_INVALID: -+ deps[i].type = ' '; -+ break; -+ case BASE_JD_DEP_TYPE_DATA: -+ deps[i].type = 'D'; -+ break; -+ case BASE_JD_DEP_TYPE_ORDER: -+ deps[i].type = '>'; -+ break; -+ default: -+ deps[i].type = '?'; -+ break; -+ } -+ } -+} -+/** -+ * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. -+ * @sfile: The debugfs entry -+ * @data: Data associated with the entry -+ * -+ * This function is called to get the contents of the JD atoms debugfs file. -+ * This is a report of all atoms managed by kbase_jd_context.atoms -+ * -+ * Return: 0 if successfully prints data in debugfs entry file, failure -+ * otherwise -+ */ -+static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) -+{ -+ struct kbase_context *kctx = sfile->private; -+ struct kbase_jd_atom *atoms; -+ unsigned long irq_flags; -+ int i; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ /* Print version */ -+ seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); -+ -+ /* Print U/K API version */ -+ seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, -+ BASE_UK_VERSION_MINOR); -+ -+ /* Print table heading */ -+ seq_puts(sfile, " ID, Core req, St, CR, Predeps, Start time, Additional info...\n"); -+ -+ atoms = kctx->jctx.atoms; -+ /* General atom states */ -+ mutex_lock(&kctx->jctx.lock); -+ /* JS-related states */ -+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); -+ for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { -+ struct kbase_jd_atom *atom = &atoms[i]; -+ s64 start_timestamp = 0; -+ struct kbase_jd_debugfs_depinfo deps[2]; -+ -+ if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) -+ continue; -+ -+ /* start_timestamp is cleared as soon as the atom leaves UNUSED state -+ * and set before a job is submitted to the h/w, a non-zero value means -+ * it is valid */ -+ if (ktime_to_ns(atom->start_timestamp)) -+ start_timestamp = ktime_to_ns( -+ ktime_sub(ktime_get(), atom->start_timestamp)); -+ -+ kbasep_jd_debugfs_atom_deps(deps, atom); -+ -+ seq_printf(sfile, -+ "%3u, %8x, %2u, %2u, %c%3u %c%3u, %20lld, ", -+ i, atom->core_req, atom->status, -+ atom->coreref_state, -+ deps[0].type, deps[0].id, -+ deps[1].type, deps[1].id, -+ start_timestamp); -+ -+ -+ kbase_jd_debugfs_fence_info(atom, sfile); -+ -+ seq_puts(sfile, "\n"); -+ } -+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); -+ mutex_unlock(&kctx->jctx.lock); -+ -+ return 0; -+} -+ -+ -+/** -+ * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file -+ * @in: &struct inode pointer -+ * @file: &struct file pointer -+ * -+ * Return: file descriptor -+ */ -+static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); -+} -+ -+static const struct file_operations kbasep_jd_debugfs_atoms_fops = { -+ .open = kbasep_jd_debugfs_atoms_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ /* Expose all atoms */ -+ debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx, -+ &kbasep_jd_debugfs_atoms_fops); -+ -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h -new file mode 100755 -index 000000000..0935f1db7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h -@@ -0,0 +1,39 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file mali_kbase_jd_debugfs.h -+ * Header file for job dispatcher-related entries in debugfs -+ */ -+ -+#ifndef _KBASE_JD_DEBUGFS_H -+#define _KBASE_JD_DEBUGFS_H -+ -+#include -+ -+#include -+ -+#define MALI_JD_DEBUGFS_VERSION 2 -+ -+/** -+ * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system -+ * -+ * @kctx Pointer to kbase_context -+ */ -+void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); -+ -+#endif /*_KBASE_JD_DEBUGFS_H*/ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_jm.c b/drivers/gpu/arm/midgard/mali_kbase_jm.c -new file mode 100755 -index 000000000..0c5c6a6f7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_jm.c -@@ -0,0 +1,131 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * HW access job manager common APIs -+ */ -+ -+#include -+#include "mali_kbase_hwaccess_jm.h" -+#include "mali_kbase_jm.h" -+ -+/** -+ * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot -+ * @js on the active context. -+ * @kbdev: Device pointer -+ * @js: Job slot to run on -+ * @nr_jobs_to_submit: Number of jobs to attempt to submit -+ * -+ * Return: true if slot can still be submitted on, false if slot is now full. -+ */ -+static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, -+ int nr_jobs_to_submit) -+{ -+ struct kbase_context *kctx; -+ int i; -+ -+ kctx = kbdev->hwaccess.active_kctx; -+ -+ if (!kctx) -+ return true; -+ -+ for (i = 0; i < nr_jobs_to_submit; i++) { -+ struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); -+ -+ if (!katom) -+ return true; /* Context has no jobs on this slot */ -+ -+ kbase_backend_run_atom(kbdev, katom); -+ } -+ -+ return false; /* Slot ringbuffer should now be full */ -+} -+ -+u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) -+{ -+ u32 ret_mask = 0; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ while (js_mask) { -+ int js = ffs(js_mask) - 1; -+ int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); -+ -+ if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) -+ ret_mask |= (1 << js); -+ -+ js_mask &= ~(1 << js); -+ } -+ -+ return ret_mask; -+} -+ -+void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!down_trylock(&js_devdata->schedule_sem)) { -+ kbase_jm_kick(kbdev, js_mask); -+ up(&js_devdata->schedule_sem); -+ } -+} -+ -+void kbase_jm_try_kick_all(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!down_trylock(&js_devdata->schedule_sem)) { -+ kbase_jm_kick_all(kbdev); -+ up(&js_devdata->schedule_sem); -+ } -+} -+ -+void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (kbdev->hwaccess.active_kctx == kctx) -+ kbdev->hwaccess.active_kctx = NULL; -+} -+ -+struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (katom->event_code != BASE_JD_EVENT_STOPPED && -+ katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { -+ return kbase_js_complete_atom(katom, NULL); -+ } else { -+ kbase_js_unpull(katom->kctx, katom); -+ return NULL; -+ } -+} -+ -+struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, ktime_t *end_timestamp) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return kbase_js_complete_atom(katom, end_timestamp); -+} -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_jm.h b/drivers/gpu/arm/midgard/mali_kbase_jm.h -new file mode 100755 -index 000000000..a74ee24c8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_jm.h -@@ -0,0 +1,110 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+/* -+ * Job manager common APIs -+ */ -+ -+#ifndef _KBASE_JM_H_ -+#define _KBASE_JM_H_ -+ -+/** -+ * kbase_jm_kick() - Indicate that there are jobs ready to run. -+ * @kbdev: Device pointer -+ * @js_mask: Mask of the job slots that can be pulled from. -+ * -+ * Caller must hold the hwaccess_lock and schedule_sem semaphore -+ * -+ * Return: Mask of the job slots that can still be submitted to. -+ */ -+u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); -+ -+/** -+ * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job -+ * slots. -+ * @kbdev: Device pointer -+ * -+ * Caller must hold the hwaccess_lock and schedule_sem semaphore -+ * -+ * Return: Mask of the job slots that can still be submitted to. -+ */ -+static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) -+{ -+ return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); -+} -+ -+/** -+ * kbase_jm_try_kick - Attempt to call kbase_jm_kick -+ * @kbdev: Device pointer -+ * @js_mask: Mask of the job slots that can be pulled from -+ * Context: Caller must hold hwaccess_lock -+ * -+ * If schedule_sem can be immediately obtained then this function will call -+ * kbase_jm_kick() otherwise it will do nothing. -+ */ -+void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); -+ -+/** -+ * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all -+ * @kbdev: Device pointer -+ * Context: Caller must hold hwaccess_lock -+ * -+ * If schedule_sem can be immediately obtained then this function will call -+ * kbase_jm_kick_all() otherwise it will do nothing. -+ */ -+void kbase_jm_try_kick_all(struct kbase_device *kbdev); -+ -+/** -+ * kbase_jm_idle_ctx() - Mark a context as idle. -+ * @kbdev: Device pointer -+ * @kctx: Context to mark as idle -+ * -+ * No more atoms will be pulled from this context until it is marked as active -+ * by kbase_js_use_ctx(). -+ * -+ * The context should have no atoms currently pulled from it -+ * (kctx->atoms_pulled == 0). -+ * -+ * Caller must hold the hwaccess_lock -+ */ -+void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has -+ * been soft-stopped or will fail due to a -+ * dependency -+ * @kbdev: Device pointer -+ * @katom: Atom that has been stopped or will be failed -+ * -+ * Return: Atom that has now been unblocked and can now be run, or NULL if none -+ */ -+struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_jm_complete() - Complete an atom -+ * @kbdev: Device pointer -+ * @katom: Atom that has completed -+ * @end_timestamp: Timestamp of atom completion -+ * -+ * Return: Atom that has now been unblocked and can now be run, or NULL if none -+ */ -+struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom, ktime_t *end_timestamp); -+ -+#endif /* _KBASE_JM_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.c b/drivers/gpu/arm/midgard/mali_kbase_js.c -new file mode 100755 -index 000000000..10a1d5909 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_js.c -@@ -0,0 +1,2834 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+/* #define ENABLE_DEBUG_LOG */ -+#include "./platform/rk/custom_log.h" -+ -+/* -+ * Job Scheduler Implementation -+ */ -+#include -+#include -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+#include -+#endif -+#include -+#include -+#include -+ -+#include -+#include -+ -+#include "mali_kbase_jm.h" -+#include "mali_kbase_hwaccess_jm.h" -+ -+/* -+ * Private types -+ */ -+ -+/* Bitpattern indicating the result of releasing a context */ -+enum { -+ /* The context was descheduled - caller should try scheduling in a new -+ * one to keep the runpool full */ -+ KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), -+ /* Ctx attributes were changed - caller should try scheduling all -+ * contexts */ -+ KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) -+}; -+ -+typedef u32 kbasep_js_release_result; -+ -+const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { -+ KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ -+ KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ -+ KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ -+}; -+ -+const base_jd_prio -+kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { -+ BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ -+ BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ -+ BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ -+}; -+ -+ -+/* -+ * Private function prototypes -+ */ -+static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( -+ struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbasep_js_atom_retained_state *katom_retained_state); -+ -+static int kbase_js_get_slot(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, -+ kbasep_js_ctx_job_cb callback); -+ -+/* Helper for trace subcodes */ -+#if KBASE_TRACE_ENABLE -+static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ return atomic_read(&kctx->refcount); -+} -+#else /* KBASE_TRACE_ENABLE */ -+static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ CSTD_UNUSED(kbdev); -+ CSTD_UNUSED(kctx); -+ return 0; -+} -+#endif /* KBASE_TRACE_ENABLE */ -+ -+/* -+ * Private functions -+ */ -+ -+/** -+ * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements -+ * @features: JSn_FEATURE register value -+ * -+ * Given a JSn_FEATURE register value returns the core requirements that match -+ * -+ * Return: Core requirement bit mask -+ */ -+static base_jd_core_req core_reqs_from_jsn_features(u16 features) -+{ -+ base_jd_core_req core_req = 0u; -+ -+ if ((features & JS_FEATURE_SET_VALUE_JOB) != 0) -+ core_req |= BASE_JD_REQ_V; -+ -+ if ((features & JS_FEATURE_CACHE_FLUSH_JOB) != 0) -+ core_req |= BASE_JD_REQ_CF; -+ -+ if ((features & JS_FEATURE_COMPUTE_JOB) != 0) -+ core_req |= BASE_JD_REQ_CS; -+ -+ if ((features & JS_FEATURE_TILER_JOB) != 0) -+ core_req |= BASE_JD_REQ_T; -+ -+ if ((features & JS_FEATURE_FRAGMENT_JOB) != 0) -+ core_req |= BASE_JD_REQ_FS; -+ -+ return core_req; -+} -+ -+static void kbase_js_sync_timers(struct kbase_device *kbdev) -+{ -+ mutex_lock(&kbdev->js_data.runpool_mutex); -+ kbase_backend_ctx_count_changed(kbdev); -+ mutex_unlock(&kbdev->js_data.runpool_mutex); -+} -+ -+/* Hold the mmu_hw_mutex and hwaccess_lock for this */ -+bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ bool result = false; -+ int as_nr; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ as_nr = kctx->as_nr; -+ if (atomic_read(&kctx->refcount) > 0) { -+ KBASE_DEBUG_ASSERT(as_nr >= 0); -+ -+ kbase_ctx_sched_retain_ctx_refcount(kctx); -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RETAIN_CTX_NOLOCK, kctx, -+ NULL, 0u, atomic_read(&kctx->refcount)); -+ result = true; -+ } -+ -+ return result; -+} -+ -+/** -+ * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms -+ * @kctx: Pointer to kbase context with ring buffer. -+ * @js: Job slot id to check. -+ * @prio: Priority to check. -+ * -+ * Return true if there are no atoms to pull. There may be running atoms in the -+ * ring buffer even if there are no atoms to pull. It is also possible for the -+ * ring buffer to be full (with running atoms) when this functions returns -+ * true. -+ * -+ * Return: true if there are no atoms to pull, false otherwise. -+ */ -+static inline bool -+jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) -+{ -+ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ return RB_EMPTY_ROOT(&rb->runnable_tree); -+} -+ -+/** -+ * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no -+ * pullable atoms -+ * @kctx: Pointer to kbase context with ring buffer. -+ * @js: Job slot id to check. -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if the ring buffers for all priorities have no pullable atoms, -+ * false otherwise. -+ */ -+static inline bool -+jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) -+{ -+ int prio; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { -+ if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) -+ return false; -+ } -+ -+ return true; -+} -+ -+/** -+ * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. -+ * @kctx: Pointer to kbase context with the queue. -+ * @js: Job slot id to iterate. -+ * @prio: Priority id to iterate. -+ * @callback: Function pointer to callback. -+ * -+ * Iterate over a queue and invoke @callback for each entry in the queue, and -+ * remove the entry from the queue. -+ * -+ * If entries are added to the queue while this is running those entries may, or -+ * may not be covered. To ensure that all entries in the buffer have been -+ * enumerated when this function returns jsctx->lock must be held when calling -+ * this function. -+ * -+ * The HW access lock must always be held when calling this function. -+ */ -+static void -+jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, -+ kbasep_js_ctx_job_cb callback) -+{ -+ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { -+ struct rb_node *node = rb_first(&queue->runnable_tree); -+ struct kbase_jd_atom *entry = rb_entry(node, -+ struct kbase_jd_atom, runnable_tree_node); -+ -+ rb_erase(node, &queue->runnable_tree); -+ callback(kctx->kbdev, entry); -+ } -+ -+ while (!list_empty(&queue->x_dep_head)) { -+ struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, -+ struct kbase_jd_atom, queue); -+ -+ list_del(queue->x_dep_head.next); -+ -+ callback(kctx->kbdev, entry); -+ } -+} -+ -+/** -+ * jsctx_queue_foreach(): - Execute callback for each entry in every queue -+ * @kctx: Pointer to kbase context with queue. -+ * @js: Job slot id to iterate. -+ * @callback: Function pointer to callback. -+ * -+ * Iterate over all the different priorities, and for each call -+ * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback -+ * for each entry, and remove the entry from the queue. -+ */ -+static inline void -+jsctx_queue_foreach(struct kbase_context *kctx, int js, -+ kbasep_js_ctx_job_cb callback) -+{ -+ int prio; -+ -+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) -+ jsctx_queue_foreach_prio(kctx, js, prio, callback); -+} -+ -+/** -+ * jsctx_rb_peek_prio(): - Check buffer and get next atom -+ * @kctx: Pointer to kbase context with ring buffer. -+ * @js: Job slot id to check. -+ * @prio: Priority id to check. -+ * -+ * Check the ring buffer for the specified @js and @prio and return a pointer to -+ * the next atom, unless the ring buffer is empty. -+ * -+ * Return: Pointer to next atom in buffer, or NULL if there is no atom. -+ */ -+static inline struct kbase_jd_atom * -+jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) -+{ -+ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; -+ struct rb_node *node; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ node = rb_first(&rb->runnable_tree); -+ if (!node) -+ return NULL; -+ -+ return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); -+} -+ -+/** -+ * jsctx_rb_peek(): - Check all priority buffers and get next atom -+ * @kctx: Pointer to kbase context with ring buffer. -+ * @js: Job slot id to check. -+ * -+ * Check the ring buffers for all priorities, starting from -+ * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a -+ * pointer to the next atom, unless all the priority's ring buffers are empty. -+ * -+ * Caller must hold the hwaccess_lock. -+ * -+ * Return: Pointer to next atom in buffer, or NULL if there is no atom. -+ */ -+static inline struct kbase_jd_atom * -+jsctx_rb_peek(struct kbase_context *kctx, int js) -+{ -+ int prio; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { -+ struct kbase_jd_atom *katom; -+ -+ katom = jsctx_rb_peek_prio(kctx, js, prio); -+ if (katom) -+ return katom; -+ } -+ -+ return NULL; -+} -+ -+/** -+ * jsctx_rb_pull(): - Mark atom in list as running -+ * @kctx: Pointer to kbase context with ring buffer. -+ * @katom: Pointer to katom to pull. -+ * -+ * Mark an atom previously obtained from jsctx_rb_peek() as running. -+ * -+ * @katom must currently be at the head of the ring buffer. -+ */ -+static inline void -+jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ int prio = katom->sched_priority; -+ int js = katom->slot_nr; -+ struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ /* Atoms must be pulled in the correct order. */ -+ WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); -+ -+ rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); -+} -+ -+#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) -+ -+static void -+jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ int prio = katom->sched_priority; -+ int js = katom->slot_nr; -+ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; -+ struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ while (*new) { -+ struct kbase_jd_atom *entry = container_of(*new, -+ struct kbase_jd_atom, runnable_tree_node); -+ -+ parent = *new; -+ if (LESS_THAN_WRAP(katom->age, entry->age)) -+ new = &((*new)->rb_left); -+ else -+ new = &((*new)->rb_right); -+ } -+ -+ /* Add new node and rebalance tree. */ -+ rb_link_node(&katom->runnable_tree_node, parent, new); -+ rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); -+} -+ -+/** -+ * jsctx_rb_unpull(): - Undo marking of atom in list as running -+ * @kctx: Pointer to kbase context with ring buffer. -+ * @katom: Pointer to katom to unpull. -+ * -+ * Undo jsctx_rb_pull() and put @katom back in the queue. -+ * -+ * jsctx_rb_unpull() must be called on atoms in the same order the atoms were -+ * pulled. -+ */ -+static inline void -+jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ jsctx_tree_add(kctx, katom); -+} -+ -+static bool kbase_js_ctx_pullable(struct kbase_context *kctx, -+ int js, -+ bool is_scheduled); -+static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js); -+static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js); -+ -+/* -+ * Functions private to KBase ('Protected' functions) -+ */ -+int kbasep_js_devdata_init(struct kbase_device * const kbdev) -+{ -+ struct kbasep_js_device_data *jsdd; -+ int i; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ jsdd = &kbdev->js_data; -+ -+#ifdef CONFIG_MALI_DEBUG -+ /* Soft-stop will be disabled on a single context by default unless -+ * softstop_always is set */ -+ jsdd->softstop_always = false; -+#endif /* CONFIG_MALI_DEBUG */ -+ jsdd->nr_all_contexts_running = 0; -+ jsdd->nr_user_contexts_running = 0; -+ jsdd->nr_contexts_pullable = 0; -+ atomic_set(&jsdd->nr_contexts_runnable, 0); -+ /* No ctx allowed to submit */ -+ jsdd->runpool_irq.submit_allowed = 0u; -+ memset(jsdd->runpool_irq.ctx_attr_ref_count, 0, -+ sizeof(jsdd->runpool_irq.ctx_attr_ref_count)); -+ memset(jsdd->runpool_irq.slot_affinities, 0, -+ sizeof(jsdd->runpool_irq.slot_affinities)); -+ memset(jsdd->runpool_irq.slot_affinity_refcount, 0, -+ sizeof(jsdd->runpool_irq.slot_affinity_refcount)); -+ INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); -+ -+ /* Config attributes */ -+ jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; -+ jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; -+ jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) -+ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408; -+ else -+ jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; -+ jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; -+ jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) -+ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408; -+ else -+ jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; -+ jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; -+ jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; -+ jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; -+ atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); -+ -+ dev_dbg(kbdev->dev, "JS Config Attribs: "); -+ dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", -+ jsdd->scheduling_period_ns); -+ dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", -+ jsdd->soft_stop_ticks); -+ dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", -+ jsdd->soft_stop_ticks_cl); -+ dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", -+ jsdd->hard_stop_ticks_ss); -+ dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", -+ jsdd->hard_stop_ticks_cl); -+ dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", -+ jsdd->hard_stop_ticks_dumping); -+ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", -+ jsdd->gpu_reset_ticks_ss); -+ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", -+ jsdd->gpu_reset_ticks_cl); -+ dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", -+ jsdd->gpu_reset_ticks_dumping); -+ dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", -+ jsdd->ctx_timeslice_ns); -+ dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", -+ atomic_read(&jsdd->soft_job_timeout_ms)); -+ -+ if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && -+ jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && -+ jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && -+ jsdd->hard_stop_ticks_dumping < -+ jsdd->gpu_reset_ticks_dumping)) { -+ dev_err(kbdev->dev, "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); -+ return -EINVAL; -+ } -+ -+#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS -+ dev_dbg(kbdev->dev, "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", -+ jsdd->soft_stop_ticks, -+ jsdd->scheduling_period_ns); -+#endif -+#if KBASE_DISABLE_SCHEDULING_HARD_STOPS -+ dev_dbg(kbdev->dev, "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", -+ jsdd->hard_stop_ticks_ss, -+ jsdd->hard_stop_ticks_dumping, -+ jsdd->scheduling_period_ns); -+#endif -+#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS -+ dev_dbg(kbdev->dev, "Note: The JS tick timer (if coded) will still be run, but do nothing."); -+#endif -+ -+ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) -+ jsdd->js_reqs[i] = core_reqs_from_jsn_features( -+ kbdev->gpu_props.props.raw_props.js_features[i]); -+ -+ /* On error, we could continue on: providing none of the below resources -+ * rely on the ones above */ -+ -+ mutex_init(&jsdd->runpool_mutex); -+ mutex_init(&jsdd->queue_mutex); -+ spin_lock_init(&kbdev->hwaccess_lock); -+ sema_init(&jsdd->schedule_sem, 1); -+ -+ for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { -+ INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i]); -+ INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i]); -+ } -+ -+ return 0; -+} -+ -+void kbasep_js_devdata_halt(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+void kbasep_js_devdata_term(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { 0, }; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ js_devdata = &kbdev->js_data; -+ -+ /* The caller must de-register all contexts before calling this -+ */ -+ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); -+ KBASE_DEBUG_ASSERT(memcmp( -+ js_devdata->runpool_irq.ctx_attr_ref_count, -+ zero_ctx_attr_ref_count, -+ sizeof(zero_ctx_attr_ref_count)) == 0); -+ CSTD_UNUSED(zero_ctx_attr_ref_count); -+} -+ -+int kbasep_js_kctx_init(struct kbase_context * const kctx) -+{ -+ struct kbase_device *kbdev; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ int i, j; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ kbdev = kctx->kbdev; -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) -+ INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); -+ -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ js_kctx_info->ctx.nr_jobs = 0; -+ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); -+ kbase_ctx_flag_clear(kctx, KCTX_DYING); -+ memset(js_kctx_info->ctx.ctx_attr_ref_count, 0, -+ sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); -+ -+ /* Initially, the context is disabled from submission until the create -+ * flags are set */ -+ kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); -+ -+ /* On error, we could continue on: providing none of the below resources -+ * rely on the ones above */ -+ mutex_init(&js_kctx_info->ctx.jsctx_mutex); -+ -+ init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); -+ -+ for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { -+ for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { -+ INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); -+ kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; -+ } -+ } -+ -+ return 0; -+} -+ -+void kbasep_js_kctx_term(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ int js; -+ bool update_ctx_count = false; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ kbdev = kctx->kbdev; -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ /* The caller must de-register all jobs before calling this */ -+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); -+ -+ mutex_lock(&kbdev->js_data.queue_mutex); -+ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) -+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); -+ -+ if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { -+ WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); -+ atomic_dec(&kbdev->js_data.nr_contexts_runnable); -+ update_ctx_count = true; -+ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); -+ } -+ -+ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ mutex_unlock(&kbdev->js_data.queue_mutex); -+ -+ if (update_ctx_count) { -+ mutex_lock(&kbdev->js_data.runpool_mutex); -+ kbase_backend_ctx_count_changed(kbdev); -+ mutex_unlock(&kbdev->js_data.runpool_mutex); -+ } -+} -+ -+/** -+ * kbase_js_ctx_list_add_pullable_nolock - Variant of -+ * kbase_jd_ctx_list_add_pullable() -+ * where the caller must hold -+ * hwaccess_lock -+ * @kbdev: Device pointer -+ * @kctx: Context to add to queue -+ * @js: Job slot to use -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if caller should call kbase_backend_ctx_count_changed() -+ */ -+static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js) -+{ -+ bool ret = false; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) -+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); -+ -+ list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], -+ &kbdev->js_data.ctx_list_pullable[js]); -+ -+ if (!kctx->slots_pullable) { -+ kbdev->js_data.nr_contexts_pullable++; -+ ret = true; -+ if (!atomic_read(&kctx->atoms_pulled)) { -+ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); -+ atomic_inc(&kbdev->js_data.nr_contexts_runnable); -+ } -+ } -+ kctx->slots_pullable |= (1 << js); -+ -+ return ret; -+} -+ -+/** -+ * kbase_js_ctx_list_add_pullable_head_nolock - Variant of -+ * kbase_js_ctx_list_add_pullable_head() -+ * where the caller must hold -+ * hwaccess_lock -+ * @kbdev: Device pointer -+ * @kctx: Context to add to queue -+ * @js: Job slot to use -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if caller should call kbase_backend_ctx_count_changed() -+ */ -+static bool kbase_js_ctx_list_add_pullable_head_nolock( -+ struct kbase_device *kbdev, struct kbase_context *kctx, int js) -+{ -+ bool ret = false; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) -+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); -+ -+ list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], -+ &kbdev->js_data.ctx_list_pullable[js]); -+ -+ if (!kctx->slots_pullable) { -+ kbdev->js_data.nr_contexts_pullable++; -+ ret = true; -+ if (!atomic_read(&kctx->atoms_pulled)) { -+ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); -+ atomic_inc(&kbdev->js_data.nr_contexts_runnable); -+ } -+ } -+ kctx->slots_pullable |= (1 << js); -+ -+ return ret; -+} -+ -+/** -+ * kbase_js_ctx_list_add_pullable_head - Add context to the head of the -+ * per-slot pullable context queue -+ * @kbdev: Device pointer -+ * @kctx: Context to add to queue -+ * @js: Job slot to use -+ * -+ * If the context is on either the pullable or unpullable queues, then it is -+ * removed before being added to the head. -+ * -+ * This function should be used when a context has been scheduled, but no jobs -+ * can currently be pulled from it. -+ * -+ * Return: true if caller should call kbase_backend_ctx_count_changed() -+ */ -+static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js) -+{ -+ bool ret; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return ret; -+} -+ -+/** -+ * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the -+ * per-slot unpullable context queue -+ * @kbdev: Device pointer -+ * @kctx: Context to add to queue -+ * @js: Job slot to use -+ * -+ * The context must already be on the per-slot pullable queue. It will be -+ * removed from the pullable queue before being added to the unpullable queue. -+ * -+ * This function should be used when a context has been pulled from, and there -+ * are no jobs remaining on the specified slot. -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if caller should call kbase_backend_ctx_count_changed() -+ */ -+static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js) -+{ -+ bool ret = false; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], -+ &kbdev->js_data.ctx_list_unpullable[js]); -+ -+ if (kctx->slots_pullable == (1 << js)) { -+ kbdev->js_data.nr_contexts_pullable--; -+ ret = true; -+ if (!atomic_read(&kctx->atoms_pulled)) { -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); -+ atomic_dec(&kbdev->js_data.nr_contexts_runnable); -+ } -+ } -+ kctx->slots_pullable &= ~(1 << js); -+ -+ return ret; -+} -+ -+/** -+ * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable -+ * or unpullable context queues -+ * @kbdev: Device pointer -+ * @kctx: Context to remove from queue -+ * @js: Job slot to use -+ * -+ * The context must already be on one of the queues. -+ * -+ * This function should be used when a context has no jobs on the GPU, and no -+ * jobs remaining for the specified slot. -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if caller should call kbase_backend_ctx_count_changed() -+ */ -+static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ int js) -+{ -+ bool ret = false; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); -+ -+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); -+ -+ if (kctx->slots_pullable == (1 << js)) { -+ kbdev->js_data.nr_contexts_pullable--; -+ ret = true; -+ if (!atomic_read(&kctx->atoms_pulled)) { -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); -+ atomic_dec(&kbdev->js_data.nr_contexts_runnable); -+ } -+ } -+ kctx->slots_pullable &= ~(1 << js); -+ -+ return ret; -+} -+ -+/** -+ * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() -+ * where the caller must hold -+ * hwaccess_lock -+ * @kbdev: Device pointer -+ * @js: Job slot to use -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: Context to use for specified slot. -+ * NULL if no contexts present for specified slot -+ */ -+static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( -+ struct kbase_device *kbdev, -+ int js) -+{ -+ struct kbase_context *kctx; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (list_empty(&kbdev->js_data.ctx_list_pullable[js])) -+ return NULL; -+ -+ kctx = list_entry(kbdev->js_data.ctx_list_pullable[js].next, -+ struct kbase_context, -+ jctx.sched_info.ctx.ctx_list_entry[js]); -+ -+ list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); -+ -+ return kctx; -+} -+ -+/** -+ * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable -+ * queue. -+ * @kbdev: Device pointer -+ * @js: Job slot to use -+ * -+ * Return: Context to use for specified slot. -+ * NULL if no contexts present for specified slot -+ */ -+static struct kbase_context *kbase_js_ctx_list_pop_head( -+ struct kbase_device *kbdev, int js) -+{ -+ struct kbase_context *kctx; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return kctx; -+} -+ -+/** -+ * kbase_js_ctx_pullable - Return if a context can be pulled from on the -+ * specified slot -+ * @kctx: Context pointer -+ * @js: Job slot to use -+ * @is_scheduled: true if the context is currently scheduled -+ * -+ * Caller must hold hwaccess_lock -+ * -+ * Return: true if context can be pulled from on specified slot -+ * false otherwise -+ */ -+static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, -+ bool is_scheduled) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbase_jd_atom *katom; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ js_devdata = &kctx->kbdev->js_data; -+ -+ if (is_scheduled) { -+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) -+ return false; -+ } -+ katom = jsctx_rb_peek(kctx, js); -+ if (!katom) -+ return false; /* No pullable atoms */ -+ if (kctx->blocked_js[js][katom->sched_priority]) -+ return false; -+ if (atomic_read(&katom->blocked)) -+ return false; /* next atom blocked */ -+ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { -+ if (katom->x_pre_dep->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || -+ katom->x_pre_dep->will_fail_event_code) -+ return false; -+ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && -+ kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) -+ return false; -+ } -+ -+ return true; -+} -+ -+static bool kbase_js_dep_validate(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ bool ret = true; -+ bool has_dep = false, has_x_dep = false; -+ int js = kbase_js_get_slot(kbdev, katom); -+ int prio = katom->sched_priority; -+ int i; -+ -+ for (i = 0; i < 2; i++) { -+ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; -+ -+ if (dep_atom) { -+ int dep_js = kbase_js_get_slot(kbdev, dep_atom); -+ int dep_prio = dep_atom->sched_priority; -+ -+ /* Dependent atom must already have been submitted */ -+ if (!(dep_atom->atom_flags & -+ KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { -+ ret = false; -+ break; -+ } -+ -+ /* Dependencies with different priorities can't -+ be represented in the ringbuffer */ -+ if (prio != dep_prio) { -+ ret = false; -+ break; -+ } -+ -+ if (js == dep_js) { -+ /* Only one same-slot dependency can be -+ * represented in the ringbuffer */ -+ if (has_dep) { -+ ret = false; -+ break; -+ } -+ /* Each dependee atom can only have one -+ * same-slot dependency */ -+ if (dep_atom->post_dep) { -+ ret = false; -+ break; -+ } -+ has_dep = true; -+ } else { -+ /* Only one cross-slot dependency can be -+ * represented in the ringbuffer */ -+ if (has_x_dep) { -+ ret = false; -+ break; -+ } -+ /* Each dependee atom can only have one -+ * cross-slot dependency */ -+ if (dep_atom->x_post_dep) { -+ ret = false; -+ break; -+ } -+ /* The dependee atom can not already be in the -+ * HW access ringbuffer */ -+ if (dep_atom->gpu_rb_state != -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { -+ ret = false; -+ break; -+ } -+ /* The dependee atom can not already have -+ * completed */ -+ if (dep_atom->status != -+ KBASE_JD_ATOM_STATE_IN_JS) { -+ ret = false; -+ break; -+ } -+ /* Cross-slot dependencies must not violate -+ * PRLAM-8987 affinity restrictions */ -+ if (kbase_hw_has_issue(kbdev, -+ BASE_HW_ISSUE_8987) && -+ (js == 2 || dep_js == 2)) { -+ ret = false; -+ break; -+ } -+ has_x_dep = true; -+ } -+ -+ /* Dependency can be represented in ringbuffers */ -+ } -+ } -+ -+ /* If dependencies can be represented by ringbuffer then clear them from -+ * atom structure */ -+ if (ret) { -+ for (i = 0; i < 2; i++) { -+ struct kbase_jd_atom *dep_atom = katom->dep[i].atom; -+ -+ if (dep_atom) { -+ int dep_js = kbase_js_get_slot(kbdev, dep_atom); -+ -+ if ((js != dep_js) && -+ (dep_atom->status != -+ KBASE_JD_ATOM_STATE_COMPLETED) -+ && (dep_atom->status != -+ KBASE_JD_ATOM_STATE_HW_COMPLETED) -+ && (dep_atom->status != -+ KBASE_JD_ATOM_STATE_UNUSED)) { -+ -+ katom->atom_flags |= -+ KBASE_KATOM_FLAG_X_DEP_BLOCKED; -+ katom->x_pre_dep = dep_atom; -+ dep_atom->x_post_dep = katom; -+ if (kbase_jd_katom_dep_type( -+ &katom->dep[i]) == -+ BASE_JD_DEP_TYPE_DATA) -+ katom->atom_flags |= -+ KBASE_KATOM_FLAG_FAIL_BLOCKER; -+ } -+ if ((kbase_jd_katom_dep_type(&katom->dep[i]) -+ == BASE_JD_DEP_TYPE_DATA) && -+ (js == dep_js)) { -+ katom->pre_dep = dep_atom; -+ dep_atom->post_dep = katom; -+ } -+ -+ list_del(&katom->dep_item[i]); -+ kbase_jd_katom_dep_clear(&katom->dep[i]); -+ } -+ } -+ } -+ -+ return ret; -+} -+ -+bool kbasep_js_add_job(struct kbase_context *kctx, -+ struct kbase_jd_atom *atom) -+{ -+ unsigned long flags; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbase_device *kbdev; -+ struct kbasep_js_device_data *js_devdata; -+ -+ bool enqueue_required = false; -+ bool timer_sync = false; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(atom != NULL); -+ lockdep_assert_held(&kctx->jctx.lock); -+ -+ kbdev = kctx->kbdev; -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ mutex_lock(&js_devdata->queue_mutex); -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ /* -+ * Begin Runpool transaction -+ */ -+ mutex_lock(&js_devdata->runpool_mutex); -+ -+ /* Refcount ctx.nr_jobs */ -+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); -+ ++(js_kctx_info->ctx.nr_jobs); -+ -+ /* Setup any scheduling information */ -+ kbasep_js_clear_job_retry_submit(atom); -+ -+ /* Lock for state available during IRQ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if (!kbase_js_dep_validate(kctx, atom)) { -+ /* Dependencies could not be represented */ -+ --(js_kctx_info->ctx.nr_jobs); -+ -+ /* Setting atom status back to queued as it still has unresolved -+ * dependencies */ -+ atom->status = KBASE_JD_ATOM_STATE_QUEUED; -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ goto out_unlock; -+ } -+ -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, TL_ATOM_STATE_READY); -+ KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom)); -+ -+ enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); -+ -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, -+ kbasep_js_trace_get_refcnt(kbdev, kctx)); -+ -+ /* Context Attribute Refcounting */ -+ kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); -+ -+ if (enqueue_required) { -+ if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) -+ timer_sync = kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, atom->slot_nr); -+ else -+ timer_sync = kbase_js_ctx_list_add_unpullable_nolock( -+ kbdev, kctx, atom->slot_nr); -+ } -+ /* If this context is active and the atom is the first on its slot, -+ * kick the job manager to attempt to fast-start the atom */ -+ if (enqueue_required && kctx == kbdev->hwaccess.active_kctx) -+ kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ if (timer_sync) -+ kbase_backend_ctx_count_changed(kbdev); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ /* End runpool transaction */ -+ -+ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { -+ if (kbase_ctx_flag(kctx, KCTX_DYING)) { -+ /* A job got added while/after kbase_job_zap_context() -+ * was called on a non-scheduled context (e.g. KDS -+ * dependency resolved). Kill that job by killing the -+ * context. */ -+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, -+ false); -+ } else if (js_kctx_info->ctx.nr_jobs == 1) { -+ /* Handle Refcount going from 0 to 1: schedule the -+ * context on the Queue */ -+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); -+ -+ /* Queue was updated - caller must try to -+ * schedule the head context */ -+ WARN_ON(!enqueue_required); -+ } -+ } -+out_unlock: -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ mutex_unlock(&js_devdata->queue_mutex); -+ -+ return enqueue_required; -+} -+ -+void kbasep_js_remove_job(struct kbase_device *kbdev, -+ struct kbase_context *kctx, struct kbase_jd_atom *atom) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbasep_js_device_data *js_devdata; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(atom != NULL); -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, -+ kbasep_js_trace_get_refcnt(kbdev, kctx)); -+ -+ /* De-refcount ctx.nr_jobs */ -+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); -+ --(js_kctx_info->ctx.nr_jobs); -+} -+ -+bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, -+ struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ unsigned long flags; -+ struct kbasep_js_atom_retained_state katom_retained_state; -+ struct kbasep_js_device_data *js_devdata; -+ bool attr_state_changed; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(katom != NULL); -+ -+ js_devdata = &kbdev->js_data; -+ -+ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); -+ kbasep_js_remove_job(kbdev, kctx, katom); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* The atom has 'finished' (will not be re-run), so no need to call -+ * kbasep_js_has_atom_finished(). -+ * -+ * This is because it returns false for soft-stopped atoms, but we -+ * want to override that, because we're cancelling an atom regardless of -+ * whether it was soft-stopped or not */ -+ attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, -+ &katom_retained_state); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return attr_state_changed; -+} -+ -+bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ unsigned long flags; -+ struct kbasep_js_device_data *js_devdata; -+ bool result; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ js_devdata = &kbdev->js_data; -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ result = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ return result; -+} -+ -+struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, -+ int as_nr) -+{ -+ int ret = 0; -+ unsigned long flags; -+ struct kbasep_js_device_data *js_devdata; -+ struct kbase_context *found_kctx = NULL; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); -+ js_devdata = &kbdev->js_data; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ found_kctx = kbdev->as_to_kctx[as_nr]; -+ -+ if (found_kctx != NULL) { -+ ret = kbase_ctx_sched_retain_ctx_refcount(found_kctx); -+ if (ret != 0) { -+ E("fail to retain ctx_refcount, ret : %d.", ret); -+ found_kctx = NULL; -+ } -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return found_kctx; -+} -+ -+/** -+ * kbasep_js_release_result - Try running more jobs after releasing a context -+ * and/or atom -+ * -+ * @kbdev: The kbase_device to operate on -+ * @kctx: The kbase_context to operate on -+ * @katom_retained_state: Retained state from the atom -+ * @runpool_ctx_attr_change: True if the runpool context attributes have changed -+ * -+ * This collates a set of actions that must happen whilst hwaccess_lock is held. -+ * -+ * This includes running more jobs when: -+ * - The previously released kctx caused a ctx attribute change, -+ * - The released atom caused a ctx attribute change, -+ * - Slots were previously blocked due to affinity restrictions, -+ * - Submission during IRQ handling failed. -+ * -+ * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were -+ * changed. The caller should try scheduling all contexts -+ */ -+static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( -+ struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ struct kbasep_js_atom_retained_state *katom_retained_state, -+ bool runpool_ctx_attr_change) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ kbasep_js_release_result result = 0; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(katom_retained_state != NULL); -+ js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (js_devdata->nr_user_contexts_running != 0) { -+ bool retry_submit = false; -+ int retry_jobslot = 0; -+ -+ if (katom_retained_state) -+ retry_submit = kbasep_js_get_atom_retry_submit_slot( -+ katom_retained_state, &retry_jobslot); -+ -+ if (runpool_ctx_attr_change || retry_submit) { -+ /* A change in runpool ctx attributes might mean we can -+ * run more jobs than before */ -+ result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; -+ -+ KBASE_TRACE_ADD_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, -+ kctx, NULL, 0u, retry_jobslot); -+ } -+ } -+ return result; -+} -+ -+/* -+ * Internal function to release the reference on a ctx and an atom's "retained -+ * state", only taking the runpool and as transaction mutexes -+ * -+ * This also starts more jobs running in the case of an ctx-attribute state -+ * change -+ * -+ * This does none of the followup actions for scheduling: -+ * - It does not schedule in a new context -+ * - It does not requeue or handle dying contexts -+ * -+ * For those tasks, just call kbasep_js_runpool_release_ctx() instead -+ * -+ * Requires: -+ * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr -+ * - Context has a non-zero refcount -+ * - Caller holds js_kctx_info->ctx.jsctx_mutex -+ * - Caller holds js_devdata->runpool_mutex -+ */ -+static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( -+ struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ struct kbasep_js_atom_retained_state *katom_retained_state) -+{ -+ unsigned long flags; -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ -+ kbasep_js_release_result release_result = 0u; -+ bool runpool_ctx_attr_change = false; -+ int kctx_as_nr; -+ struct kbase_as *current_as; -+ int new_ref_count; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ js_kctx_info = &kctx->jctx.sched_info; -+ js_devdata = &kbdev->js_data; -+ -+ /* Ensure context really is scheduled in */ -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ kctx_as_nr = kctx->as_nr; -+ KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); -+ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); -+ -+ /* -+ * Transaction begins on AS and runpool_irq -+ * -+ * Assert about out calling contract -+ */ -+ current_as = &kbdev->as[kctx_as_nr]; -+ mutex_lock(&kbdev->pm.lock); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); -+ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); -+ -+ /* Update refcount */ -+ kbase_ctx_sched_release_ctx(kctx); -+ new_ref_count = atomic_read(&kctx->refcount); -+ -+ /* Release the atom if it finished (i.e. wasn't soft-stopped) */ -+ if (kbasep_js_has_atom_finished(katom_retained_state)) -+ runpool_ctx_attr_change |= kbasep_js_ctx_attr_ctx_release_atom( -+ kbdev, kctx, katom_retained_state); -+ -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_RELEASE_CTX, kctx, NULL, 0u, -+ new_ref_count); -+ -+ if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && -+ !kbase_pm_is_suspending(kbdev)) { -+ /* Context is kept scheduled into an address space even when -+ * there are no jobs, in this case we have to handle the -+ * situation where all jobs have been evicted from the GPU and -+ * submission is disabled. -+ * -+ * At this point we re-enable submission to allow further jobs -+ * to be executed -+ */ -+ kbasep_js_set_submit_allowed(js_devdata, kctx); -+ } -+ -+ /* Make a set of checks to see if the context should be scheduled out. -+ * Note that there'll always be at least 1 reference to the context -+ * which was previously acquired by kbasep_js_schedule_ctx(). */ -+ if (new_ref_count == 1 && -+ (!kbasep_js_is_submit_allowed(js_devdata, kctx) || -+ kbdev->pm.suspending)) { -+ int num_slots = kbdev->gpu_props.num_job_slots; -+ int slot; -+ -+ /* Last reference, and we've been told to remove this context -+ * from the Run Pool */ -+ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", -+ kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, -+ kbasep_js_is_submit_allowed(js_devdata, kctx)); -+ -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_mmu_as_released(kctx->as_nr); -+#endif -+ KBASE_TLSTREAM_TL_NRET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); -+ -+ kbase_backend_release_ctx_irq(kbdev, kctx); -+ -+ if (kbdev->hwaccess.active_kctx == kctx) -+ kbdev->hwaccess.active_kctx = NULL; -+ -+ /* Ctx Attribute handling -+ * -+ * Releasing atoms attributes must either happen before this, or -+ * after the KCTX_SHEDULED flag is changed, otherwise we -+ * double-decount the attributes -+ */ -+ runpool_ctx_attr_change |= -+ kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); -+ -+ /* Releasing the context and katom retained state can allow -+ * more jobs to run */ -+ release_result |= -+ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, -+ kctx, katom_retained_state, -+ runpool_ctx_attr_change); -+ -+ /* -+ * Transaction ends on AS and runpool_irq: -+ * -+ * By this point, the AS-related data is now clear and ready -+ * for re-use. -+ * -+ * Since releases only occur once for each previous successful -+ * retain, and no more retains are allowed on this context, no -+ * other thread will be operating in this -+ * code whilst we are -+ */ -+ -+ /* Recalculate pullable status for all slots */ -+ for (slot = 0; slot < num_slots; slot++) { -+ if (kbase_js_ctx_pullable(kctx, slot, false)) -+ kbase_js_ctx_list_add_pullable_nolock(kbdev, -+ kctx, slot); -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ kbase_backend_release_ctx_noirq(kbdev, kctx); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ -+ /* Note: Don't reuse kctx_as_nr now */ -+ -+ /* Synchronize with any timers */ -+ kbase_backend_ctx_count_changed(kbdev); -+ -+ /* update book-keeping info */ -+ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); -+ /* Signal any waiter that the context is not scheduled, so is -+ * safe for termination - once the jsctx_mutex is also dropped, -+ * and jobs have finished. */ -+ wake_up(&js_kctx_info->ctx.is_scheduled_wait); -+ -+ /* Queue an action to occur after we've dropped the lock */ -+ release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | -+ KBASEP_JS_RELEASE_RESULT_SCHED_ALL; -+ } else { -+ kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, -+ katom_retained_state, runpool_ctx_attr_change); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->pm.lock); -+ } -+ -+ return release_result; -+} -+ -+void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_atom_retained_state katom_retained_state; -+ -+ /* Setup a dummy katom_retained_state */ -+ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); -+ -+ kbasep_js_runpool_release_ctx_internal(kbdev, kctx, -+ &katom_retained_state); -+} -+ -+void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx, bool has_pm_ref) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ js_devdata = &kbdev->js_data; -+ -+ /* This is called if and only if you've you've detached the context from -+ * the Runpool Queue, and not added it back to the Runpool -+ */ -+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ if (kbase_ctx_flag(kctx, KCTX_DYING)) { -+ /* Dying: don't requeue, but kill all jobs on the context. This -+ * happens asynchronously */ -+ dev_dbg(kbdev->dev, -+ "JS: ** Killing Context %p on RunPool Remove **", kctx); -+ kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); -+ } -+} -+ -+void kbasep_js_runpool_release_ctx_and_katom_retained_state( -+ struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbasep_js_atom_retained_state *katom_retained_state) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ kbasep_js_release_result release_result; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ js_kctx_info = &kctx->jctx.sched_info; -+ js_devdata = &kbdev->js_data; -+ -+ mutex_lock(&js_devdata->queue_mutex); -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ -+ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, -+ katom_retained_state); -+ -+ /* Drop the runpool mutex to allow requeing kctx */ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) -+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); -+ -+ /* Drop the jsctx_mutex to allow scheduling in a new context */ -+ -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ -+ if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) -+ kbase_js_sched_all(kbdev); -+} -+ -+void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_atom_retained_state katom_retained_state; -+ -+ kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); -+ -+ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, -+ &katom_retained_state); -+} -+ -+/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into -+ * kbase_js_sched_all() */ -+static void kbasep_js_runpool_release_ctx_no_schedule( -+ struct kbase_device *kbdev, struct kbase_context *kctx) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ kbasep_js_release_result release_result; -+ struct kbasep_js_atom_retained_state katom_retained_state_struct; -+ struct kbasep_js_atom_retained_state *katom_retained_state = -+ &katom_retained_state_struct; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ js_kctx_info = &kctx->jctx.sched_info; -+ js_devdata = &kbdev->js_data; -+ kbasep_js_atom_retained_state_init_invalid(katom_retained_state); -+ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ -+ release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, -+ katom_retained_state); -+ -+ /* Drop the runpool mutex to allow requeing kctx */ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) -+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); -+ -+ /* Drop the jsctx_mutex to allow scheduling in a new context */ -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ /* NOTE: could return release_result if the caller would like to know -+ * whether it should schedule a new context, but currently no callers do -+ */ -+} -+ -+void kbase_js_set_timeouts(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbase_backend_timeouts_changed(kbdev); -+} -+ -+static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbase_as *new_address_space = NULL; -+ unsigned long flags; -+ bool kctx_suspended = false; -+ int as_nr; -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ /* Pick available address space for this context */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ as_nr = kbase_ctx_sched_retain_ctx(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ if (as_nr == KBASEP_AS_NR_INVALID) { -+ as_nr = kbase_backend_find_and_release_free_address_space( -+ kbdev, kctx); -+ if (as_nr != KBASEP_AS_NR_INVALID) { -+ /* Attempt to retain the context again, this should -+ * succeed */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ as_nr = kbase_ctx_sched_retain_ctx(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ WARN_ON(as_nr == KBASEP_AS_NR_INVALID); -+ } -+ } -+ if (as_nr == KBASEP_AS_NR_INVALID) -+ return false; /* No address spaces currently available */ -+ -+ new_address_space = &kbdev->as[as_nr]; -+ -+ /* -+ * Atomic transaction on the Context and Run Pool begins -+ */ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* Check to see if context is dying due to kbase_job_zap_context() */ -+ if (kbase_ctx_flag(kctx, KCTX_DYING)) { -+ /* Roll back the transaction so far and return */ -+ kbase_ctx_sched_release_ctx(kctx); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ return false; -+ } -+ -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, -+ 0u, -+ kbasep_js_trace_get_refcnt(kbdev, kctx)); -+ -+ kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); -+ -+ /* Assign context to previously chosen address space */ -+ if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { -+ /* Roll back the transaction so far and return */ -+ kbase_ctx_sched_release_ctx(kctx); -+ kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ return false; -+ } -+ -+ kbdev->hwaccess.active_kctx = kctx; -+ -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_mmu_as_in_use(kctx->as_nr); -+#endif -+ KBASE_TLSTREAM_TL_RET_AS_CTX(&kbdev->as[kctx->as_nr], kctx); -+ -+ /* Cause any future waiter-on-termination to wait until the context is -+ * descheduled */ -+ wake_up(&js_kctx_info->ctx.is_scheduled_wait); -+ -+ /* Re-check for suspending: a suspend could've occurred, and all the -+ * contexts could've been removed from the runpool before we took this -+ * lock. In this case, we don't want to allow this context to run jobs, -+ * we just want it out immediately. -+ * -+ * The DMB required to read the suspend flag was issued recently as part -+ * of the hwaccess_lock locking. If a suspend occurs *after* that lock -+ * was taken (i.e. this condition doesn't execute), then the -+ * kbasep_js_suspend() code will cleanup this context instead (by virtue -+ * of it being called strictly after the suspend flag is set, and will -+ * wait for this lock to drop) */ -+ if (kbase_pm_is_suspending(kbdev)) { -+ /* Cause it to leave at some later point */ -+ bool retained; -+ -+ retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); -+ KBASE_DEBUG_ASSERT(retained); -+ -+ kbasep_js_clear_submit_allowed(js_devdata, kctx); -+ kctx_suspended = true; -+ } -+ -+ /* Transaction complete */ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ /* Synchronize with any timers */ -+ kbase_backend_ctx_count_changed(kbdev); -+ -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ /* Note: after this point, the context could potentially get scheduled -+ * out immediately */ -+ -+ if (kctx_suspended) { -+ /* Finishing forcing out the context due to a suspend. Use a -+ * variant of kbasep_js_runpool_release_ctx() that doesn't -+ * schedule a new context, to prevent a risk of recursion back -+ * into this function */ -+ kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); -+ return false; -+ } -+ return true; -+} -+ -+static bool kbase_js_use_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && -+ kbase_backend_use_ctx_sched(kbdev, kctx)) { -+ /* Context already has ASID - mark as active */ -+ kbdev->hwaccess.active_kctx = kctx; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ return true; /* Context already scheduled */ -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return kbasep_js_schedule_ctx(kbdev, kctx); -+} -+ -+void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbasep_js_device_data *js_devdata; -+ bool is_scheduled; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ /* This must never be attempted whilst suspending - i.e. it should only -+ * happen in response to a syscall from a user-space thread */ -+ BUG_ON(kbase_pm_is_suspending(kbdev)); -+ -+ mutex_lock(&js_devdata->queue_mutex); -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ /* Mark the context as privileged */ -+ kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); -+ -+ is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); -+ if (!is_scheduled) { -+ /* Add the context to the pullable list */ -+ if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) -+ kbase_js_sync_timers(kbdev); -+ -+ /* Fast-starting requires the jsctx_mutex to be dropped, -+ * because it works on multiple ctxs */ -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ -+ /* Try to schedule the context in */ -+ kbase_js_sched_all(kbdev); -+ -+ /* Wait for the context to be scheduled in */ -+ wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, -+ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ } else { -+ /* Already scheduled in - We need to retain it to keep the -+ * corresponding address space */ -+ kbasep_js_runpool_retain_ctx(kbdev, kctx); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ } -+} -+KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); -+ -+void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ /* We don't need to use the address space anymore */ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ /* Release the context - it will be scheduled out */ -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ -+ kbase_js_sched_all(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); -+ -+void kbasep_js_suspend(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ struct kbasep_js_device_data *js_devdata; -+ int i; -+ u16 retained = 0u; -+ int nr_privileged_ctx = 0; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); -+ js_devdata = &kbdev->js_data; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* Prevent all contexts from submitting */ -+ js_devdata->runpool_irq.submit_allowed = 0; -+ -+ /* Retain each of the contexts, so we can cause it to leave even if it -+ * had no refcount to begin with */ -+ for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { -+ struct kbase_context *kctx = kbdev->as_to_kctx[i]; -+ -+ retained = retained << 1; -+ -+ if (kctx) { -+ kbase_ctx_sched_retain_ctx_refcount(kctx); -+ retained |= 1u; -+ /* We can only cope with up to 1 privileged context - -+ * the instrumented context. It'll be suspended by -+ * disabling instrumentation */ -+ if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { -+ ++nr_privileged_ctx; -+ WARN_ON(nr_privileged_ctx != 1); -+ } -+ } -+ } -+ CSTD_UNUSED(nr_privileged_ctx); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* De-ref the previous retain to ensure each context gets pulled out -+ * sometime later. */ -+ for (i = 0; -+ i < BASE_MAX_NR_AS; -+ ++i, retained = retained >> 1) { -+ struct kbase_context *kctx = kbdev->as_to_kctx[i]; -+ -+ if (retained & 1u) -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ } -+ -+ /* Caller must wait for all Power Manager active references to be -+ * dropped */ -+} -+ -+void kbasep_js_resume(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ int js; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ js_devdata = &kbdev->js_data; -+ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); -+ -+ mutex_lock(&js_devdata->queue_mutex); -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ struct kbase_context *kctx, *n; -+ -+ list_for_each_entry_safe(kctx, n, -+ &kbdev->js_data.ctx_list_unpullable[js], -+ jctx.sched_info.ctx.ctx_list_entry[js]) { -+ struct kbasep_js_kctx_info *js_kctx_info; -+ unsigned long flags; -+ bool timer_sync = false; -+ -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_lock(&js_devdata->runpool_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && -+ kbase_js_ctx_pullable(kctx, js, false)) -+ timer_sync = -+ kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, js); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ if (timer_sync) -+ kbase_backend_ctx_count_changed(kbdev); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ } -+ } -+ mutex_unlock(&js_devdata->queue_mutex); -+ -+ /* Restart atom processing */ -+ kbase_js_sched_all(kbdev); -+ -+ /* JS Resume complete */ -+} -+ -+bool kbase_js_is_atom_valid(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ if ((katom->core_req & BASE_JD_REQ_FS) && -+ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | -+ BASE_JD_REQ_T))) -+ return false; -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) && -+ (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) && -+ (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T))) -+ return false; -+ -+ return true; -+} -+ -+static int kbase_js_get_slot(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom) -+{ -+ if (katom->core_req & BASE_JD_REQ_FS) -+ return 0; -+ -+ if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { -+ if (katom->device_nr == 1 && -+ kbdev->gpu_props.num_core_groups == 2) -+ return 2; -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) -+ return 2; -+ } -+ -+ return 1; -+} -+ -+bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom) -+{ -+ bool enqueue_required; -+ -+ katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ lockdep_assert_held(&kctx->jctx.lock); -+ -+ /* If slot will transition from unpullable to pullable then add to -+ * pullable list */ -+ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { -+ enqueue_required = true; -+ } else { -+ enqueue_required = false; -+ } -+ if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || -+ (katom->pre_dep && (katom->pre_dep->atom_flags & -+ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { -+ int prio = katom->sched_priority; -+ int js = katom->slot_nr; -+ struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; -+ -+ list_add_tail(&katom->queue, &queue->x_dep_head); -+ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; -+ enqueue_required = false; -+ } else { -+ /* Check if there are lower priority jobs to soft stop */ -+ kbase_job_slot_ctx_priority_check_locked(kctx, katom); -+ -+ /* Add atom to ring buffer. */ -+ jsctx_tree_add(kctx, katom); -+ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; -+ } -+ -+ return enqueue_required; -+} -+ -+/** -+ * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the -+ * runnable_tree, ready for execution -+ * @katom: Atom to submit -+ * -+ * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, -+ * but is still present in the x_dep list. If @katom has a same-slot dependent -+ * atom then that atom (and any dependents) will also be moved. -+ */ -+static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) -+{ -+ lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); -+ -+ while (katom) { -+ WARN_ON(!(katom->atom_flags & -+ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); -+ -+ if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { -+ list_del(&katom->queue); -+ katom->atom_flags &= -+ ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; -+ jsctx_tree_add(katom->kctx, katom); -+ katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; -+ } else { -+ break; -+ } -+ -+ katom = katom->post_dep; -+ } -+} -+ -+ -+/** -+ * kbase_js_evict_deps - Evict dependencies of a failed atom. -+ * @kctx: Context pointer -+ * @katom: Pointer to the atom that has failed. -+ * @js: The job slot the katom was run on. -+ * @prio: Priority of the katom. -+ * -+ * Remove all post dependencies of an atom from the context ringbuffers. -+ * -+ * The original atom's event_code will be propogated to all dependent atoms. -+ * -+ * Context: Caller must hold the HW access lock -+ */ -+static void kbase_js_evict_deps(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js, int prio) -+{ -+ struct kbase_jd_atom *x_dep = katom->x_post_dep; -+ struct kbase_jd_atom *next_katom = katom->post_dep; -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ if (next_katom) { -+ KBASE_DEBUG_ASSERT(next_katom->status != -+ KBASE_JD_ATOM_STATE_HW_COMPLETED); -+ next_katom->will_fail_event_code = katom->event_code; -+ -+ } -+ -+ /* Has cross slot depenency. */ -+ if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | -+ KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { -+ /* Remove dependency.*/ -+ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; -+ -+ /* Fail if it had a data dependency. */ -+ if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { -+ x_dep->will_fail_event_code = katom->event_code; -+ } -+ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) -+ kbase_js_move_to_tree(x_dep); -+ } -+} -+ -+struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) -+{ -+ struct kbase_jd_atom *katom; -+ struct kbasep_js_device_data *js_devdata; -+ struct kbase_device *kbdev; -+ int pulled; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ kbdev = kctx->kbdev; -+ -+ js_devdata = &kbdev->js_data; -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) -+ return NULL; -+ if (kbase_pm_is_suspending(kbdev)) -+ return NULL; -+ -+ katom = jsctx_rb_peek(kctx, js); -+ if (!katom) -+ return NULL; -+ if (kctx->blocked_js[js][katom->sched_priority]) -+ return NULL; -+ if (atomic_read(&katom->blocked)) -+ return NULL; -+ -+ /* Due to ordering restrictions when unpulling atoms on failure, we do -+ * not allow multiple runs of fail-dep atoms from the same context to be -+ * present on the same slot */ -+ if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { -+ struct kbase_jd_atom *prev_atom = -+ kbase_backend_inspect_tail(kbdev, js); -+ -+ if (prev_atom && prev_atom->kctx != kctx) -+ return NULL; -+ } -+ -+ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { -+ if (katom->x_pre_dep->gpu_rb_state == -+ KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || -+ katom->x_pre_dep->will_fail_event_code) -+ return NULL; -+ if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && -+ kbase_backend_nr_atoms_on_slot(kbdev, js)) -+ return NULL; -+ } -+ -+ kbase_ctx_flag_set(kctx, KCTX_PULLED); -+ -+ pulled = atomic_inc_return(&kctx->atoms_pulled); -+ if (pulled == 1 && !kctx->slots_pullable) { -+ WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); -+ atomic_inc(&kbdev->js_data.nr_contexts_runnable); -+ } -+ atomic_inc(&kctx->atoms_pulled_slot[katom->slot_nr]); -+ kctx->atoms_pulled_slot_pri[katom->slot_nr][katom->sched_priority]++; -+ jsctx_rb_pull(kctx, katom); -+ -+ kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); -+ -+ katom->atom_flags |= KBASE_KATOM_FLAG_HOLDING_CTX_REF; -+ -+ katom->ticks = 0; -+ -+ return katom; -+} -+ -+ -+static void js_return_worker(struct work_struct *data) -+{ -+ struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, -+ work); -+ struct kbase_context *kctx = katom->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; -+ struct kbasep_js_atom_retained_state retained_state; -+ int js = katom->slot_nr; -+ int prio = katom->sched_priority; -+ bool timer_sync = false; -+ bool context_idle = false; -+ unsigned long flags; -+ base_jd_core_req core_req = katom->core_req; -+ u64 affinity = katom->affinity; -+ enum kbase_atom_coreref_state coreref_state = katom->coreref_state; -+ -+ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(katom); -+ -+ kbase_backend_complete_wq(kbdev, katom); -+ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) -+ kbase_as_poking_timer_release_atom(kbdev, kctx, katom); -+ -+ kbasep_js_atom_retained_state_copy(&retained_state, katom); -+ -+ mutex_lock(&js_devdata->queue_mutex); -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ -+ atomic_dec(&kctx->atoms_pulled); -+ atomic_dec(&kctx->atoms_pulled_slot[js]); -+ -+ atomic_dec(&katom->blocked); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ kctx->atoms_pulled_slot_pri[js][katom->sched_priority]--; -+ -+ if (!atomic_read(&kctx->atoms_pulled_slot[js]) && -+ jsctx_rb_none_to_pull(kctx, js)) -+ timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); -+ -+ /* If this slot has been blocked due to soft-stopped atoms, and all -+ * atoms have now been processed, then unblock the slot */ -+ if (!kctx->atoms_pulled_slot_pri[js][prio] && -+ kctx->blocked_js[js][prio]) { -+ kctx->blocked_js[js][prio] = false; -+ -+ /* Only mark the slot as pullable if the context is not idle - -+ * that case is handled below */ -+ if (atomic_read(&kctx->atoms_pulled) && -+ kbase_js_ctx_pullable(kctx, js, true)) -+ timer_sync |= kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, js); -+ } -+ -+ if (!atomic_read(&kctx->atoms_pulled)) { -+ if (!kctx->slots_pullable) { -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); -+ atomic_dec(&kbdev->js_data.nr_contexts_runnable); -+ timer_sync = true; -+ } -+ -+ if (kctx->as_nr != KBASEP_AS_NR_INVALID && -+ !kbase_ctx_flag(kctx, KCTX_DYING)) { -+ int num_slots = kbdev->gpu_props.num_job_slots; -+ int slot; -+ -+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) -+ kbasep_js_set_submit_allowed(js_devdata, kctx); -+ -+ for (slot = 0; slot < num_slots; slot++) { -+ if (kbase_js_ctx_pullable(kctx, slot, true)) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, slot); -+ } -+ } -+ -+ kbase_jm_idle_ctx(kbdev, kctx); -+ -+ context_idle = true; -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (context_idle) { -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); -+ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); -+ kbase_pm_context_idle(kbdev); -+ } -+ -+ if (timer_sync) -+ kbase_js_sync_timers(kbdev); -+ -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ -+ katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; -+ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, -+ &retained_state); -+ -+ kbase_js_sched_all(kbdev); -+ -+ kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, -+ coreref_state); -+} -+ -+void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ jsctx_rb_unpull(kctx, katom); -+ -+ WARN_ON(work_pending(&katom->work)); -+ -+ /* Block re-submission until workqueue has run */ -+ atomic_inc(&katom->blocked); -+ -+ kbase_job_check_leave_disjoint(kctx->kbdev, katom); -+ -+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); -+ INIT_WORK(&katom->work, js_return_worker); -+ queue_work(kctx->jctx.job_done_wq, &katom->work); -+} -+ -+bool kbase_js_complete_atom_wq(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ struct kbasep_js_device_data *js_devdata; -+ struct kbase_device *kbdev; -+ unsigned long flags; -+ bool timer_sync = false; -+ int atom_slot; -+ bool context_idle = false; -+ int prio = katom->sched_priority; -+ -+ kbdev = kctx->kbdev; -+ atom_slot = katom->slot_nr; -+ -+ js_kctx_info = &kctx->jctx.sched_info; -+ js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { -+ context_idle = !atomic_dec_return(&kctx->atoms_pulled); -+ atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); -+ kctx->atoms_pulled_slot_pri[atom_slot][prio]--; -+ -+ if (!atomic_read(&kctx->atoms_pulled) && -+ !kctx->slots_pullable) { -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); -+ kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); -+ atomic_dec(&kbdev->js_data.nr_contexts_runnable); -+ timer_sync = true; -+ } -+ -+ /* If this slot has been blocked due to soft-stopped atoms, and -+ * all atoms have now been processed, then unblock the slot */ -+ if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] -+ && kctx->blocked_js[atom_slot][prio]) { -+ kctx->blocked_js[atom_slot][prio] = false; -+ if (kbase_js_ctx_pullable(kctx, atom_slot, true)) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, atom_slot); -+ } -+ } -+ WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); -+ -+ if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && -+ jsctx_rb_none_to_pull(kctx, atom_slot)) { -+ if (!list_empty( -+ &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) -+ timer_sync |= kbase_js_ctx_list_remove_nolock( -+ kctx->kbdev, kctx, atom_slot); -+ } -+ -+ /* -+ * If submission is disabled on this context (most likely due to an -+ * atom failure) and there are now no atoms left in the system then -+ * re-enable submission so that context can be scheduled again. -+ */ -+ if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && -+ !atomic_read(&kctx->atoms_pulled) && -+ !kbase_ctx_flag(kctx, KCTX_DYING)) { -+ int js; -+ -+ kbasep_js_set_submit_allowed(js_devdata, kctx); -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ if (kbase_js_ctx_pullable(kctx, js, true)) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, js); -+ } -+ } else if (katom->x_post_dep && -+ kbasep_js_is_submit_allowed(js_devdata, kctx)) { -+ int js; -+ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ if (kbase_js_ctx_pullable(kctx, js, true)) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_nolock( -+ kbdev, kctx, js); -+ } -+ } -+ -+ /* Mark context as inactive. The pm reference will be dropped later in -+ * jd_done_worker(). -+ */ -+ if (context_idle) -+ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ if (timer_sync) -+ kbase_backend_ctx_count_changed(kbdev); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ return context_idle; -+} -+ -+struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, -+ ktime_t *end_timestamp) -+{ -+ u64 microseconds_spent = 0; -+ struct kbase_device *kbdev; -+ struct kbase_context *kctx = katom->kctx; -+ struct kbase_jd_atom *x_dep = katom->x_post_dep; -+ -+ kbdev = kctx->kbdev; -+ -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ if (katom->will_fail_event_code) -+ katom->event_code = katom->will_fail_event_code; -+ -+ katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; -+ -+ if (katom->event_code != BASE_JD_EVENT_DONE) { -+ kbase_js_evict_deps(kctx, katom, katom->slot_nr, -+ katom->sched_priority); -+ } -+ -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, -+ katom->slot_nr), NULL, 0); -+#endif -+ -+ /* Calculate the job's time used */ -+ if (end_timestamp != NULL) { -+ /* Only calculating it for jobs that really run on the HW (e.g. -+ * removed from next jobs never actually ran, so really did take -+ * zero time) */ -+ ktime_t tick_diff = ktime_sub(*end_timestamp, -+ katom->start_timestamp); -+ -+ microseconds_spent = ktime_to_ns(tick_diff); -+ -+ do_div(microseconds_spent, 1000); -+ -+ /* Round up time spent to the minimum timer resolution */ -+ if (microseconds_spent < KBASEP_JS_TICK_RESOLUTION_US) -+ microseconds_spent = KBASEP_JS_TICK_RESOLUTION_US; -+ } -+ -+ -+ kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); -+ -+ /* Unblock cross dependency if present */ -+ if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || -+ !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && -+ (x_dep->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { -+ bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, -+ false); -+ x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; -+ kbase_js_move_to_tree(x_dep); -+ if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, -+ false)) -+ kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, -+ x_dep->slot_nr); -+ -+ if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) -+ return x_dep; -+ } -+ -+ return NULL; -+} -+ -+void kbase_js_sched(struct kbase_device *kbdev, int js_mask) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbase_context *last_active; -+ bool timer_sync = false; -+ bool ctx_waiting = false; -+ -+ js_devdata = &kbdev->js_data; -+ -+ down(&js_devdata->schedule_sem); -+ mutex_lock(&js_devdata->queue_mutex); -+ -+ last_active = kbdev->hwaccess.active_kctx; -+ -+ while (js_mask) { -+ int js; -+ -+ js = ffs(js_mask) - 1; -+ -+ while (1) { -+ struct kbase_context *kctx; -+ unsigned long flags; -+ bool context_idle = false; -+ -+ kctx = kbase_js_ctx_list_pop_head(kbdev, js); -+ -+ if (!kctx) { -+ js_mask &= ~(1 << js); -+ break; /* No contexts on pullable list */ -+ } -+ -+ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { -+ context_idle = true; -+ -+ if (kbase_pm_context_active_handle_suspend( -+ kbdev, -+ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { -+ /* Suspend pending - return context to -+ * queue and stop scheduling */ -+ mutex_lock( -+ &kctx->jctx.sched_info.ctx.jsctx_mutex); -+ if (kbase_js_ctx_list_add_pullable_head( -+ kctx->kbdev, kctx, js)) -+ kbase_js_sync_timers(kbdev); -+ mutex_unlock( -+ &kctx->jctx.sched_info.ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ up(&js_devdata->schedule_sem); -+ return; -+ } -+ kbase_ctx_flag_set(kctx, KCTX_ACTIVE); -+ } -+ -+ if (!kbase_js_use_ctx(kbdev, kctx)) { -+ mutex_lock( -+ &kctx->jctx.sched_info.ctx.jsctx_mutex); -+ /* Context can not be used at this time */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (kbase_js_ctx_pullable(kctx, js, false) -+ || kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_head_nolock( -+ kctx->kbdev, kctx, js); -+ else -+ timer_sync |= -+ kbase_js_ctx_list_add_unpullable_nolock( -+ kctx->kbdev, kctx, js); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, -+ flags); -+ mutex_unlock( -+ &kctx->jctx.sched_info.ctx.jsctx_mutex); -+ if (context_idle) { -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); -+ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); -+ kbase_pm_context_idle(kbdev); -+ } -+ -+ /* No more jobs can be submitted on this slot */ -+ js_mask &= ~(1 << js); -+ break; -+ } -+ mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ kbase_ctx_flag_clear(kctx, KCTX_PULLED); -+ -+ if (!kbase_jm_kick(kbdev, 1 << js)) -+ /* No more jobs can be submitted on this slot */ -+ js_mask &= ~(1 << js); -+ -+ if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { -+ bool pullable = kbase_js_ctx_pullable(kctx, js, -+ true); -+ -+ /* Failed to pull jobs - push to head of list. -+ * Unless this context is already 'active', in -+ * which case it's effectively already scheduled -+ * so push it to the back of the list. */ -+ if (pullable && kctx == last_active) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_nolock( -+ kctx->kbdev, -+ kctx, js); -+ else if (pullable) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_head_nolock( -+ kctx->kbdev, -+ kctx, js); -+ else -+ timer_sync |= -+ kbase_js_ctx_list_add_unpullable_nolock( -+ kctx->kbdev, -+ kctx, js); -+ -+ /* If this context is not the active context, -+ * but the active context is pullable on this -+ * slot, then we need to remove the active -+ * marker to prevent it from submitting atoms in -+ * the IRQ handler, which would prevent this -+ * context from making progress. */ -+ if (last_active && kctx != last_active && -+ kbase_js_ctx_pullable( -+ last_active, js, true)) -+ ctx_waiting = true; -+ -+ if (context_idle) { -+ kbase_jm_idle_ctx(kbdev, kctx); -+ spin_unlock_irqrestore( -+ &kbdev->hwaccess_lock, -+ flags); -+ WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); -+ kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); -+ kbase_pm_context_idle(kbdev); -+ } else { -+ spin_unlock_irqrestore( -+ &kbdev->hwaccess_lock, -+ flags); -+ } -+ mutex_unlock( -+ &kctx->jctx.sched_info.ctx.jsctx_mutex); -+ -+ js_mask &= ~(1 << js); -+ break; /* Could not run atoms on this slot */ -+ } -+ -+ /* Push to back of list */ -+ if (kbase_js_ctx_pullable(kctx, js, true)) -+ timer_sync |= -+ kbase_js_ctx_list_add_pullable_nolock( -+ kctx->kbdev, kctx, js); -+ else -+ timer_sync |= -+ kbase_js_ctx_list_add_unpullable_nolock( -+ kctx->kbdev, kctx, js); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); -+ } -+ } -+ -+ if (timer_sync) -+ kbase_js_sync_timers(kbdev); -+ -+ if (kbdev->hwaccess.active_kctx == last_active && ctx_waiting) -+ kbdev->hwaccess.active_kctx = NULL; -+ -+ mutex_unlock(&js_devdata->queue_mutex); -+ up(&js_devdata->schedule_sem); -+} -+ -+void kbase_js_zap_context(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; -+ int js; -+ -+ /* -+ * Critical assumption: No more submission is possible outside of the -+ * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) -+ * whilst the struct kbase_context is terminating. -+ */ -+ -+ /* First, atomically do the following: -+ * - mark the context as dying -+ * - try to evict it from the queue */ -+ mutex_lock(&kctx->jctx.lock); -+ mutex_lock(&js_devdata->queue_mutex); -+ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); -+ kbase_ctx_flag_set(kctx, KCTX_DYING); -+ -+ dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); -+ -+ /* -+ * At this point we know: -+ * - If eviction succeeded, it was in the queue, but now no -+ * longer is -+ * - We must cancel the jobs here. No Power Manager active reference to -+ * release. -+ * - This happens asynchronously - kbase_jd_zap_context() will wait for -+ * those jobs to be killed. -+ * - If eviction failed, then it wasn't in the queue. It is one -+ * of the following: -+ * - a. it didn't have any jobs, and so is not in the Queue or -+ * the Run Pool (not scheduled) -+ * - Hence, no more work required to cancel jobs. No Power Manager -+ * active reference to release. -+ * - b. it was in the middle of a scheduling transaction (and thus must -+ * have at least 1 job). This can happen from a syscall or a -+ * kernel thread. We still hold the jsctx_mutex, and so the thread -+ * must be waiting inside kbasep_js_try_schedule_head_ctx(), -+ * before checking whether the runpool is full. That thread will -+ * continue after we drop the mutex, and will notice the context -+ * is dying. It will rollback the transaction, killing all jobs at -+ * the same time. kbase_jd_zap_context() will wait for those jobs -+ * to be killed. -+ * - Hence, no more work required to cancel jobs, or to release the -+ * Power Manager active reference. -+ * - c. it is scheduled, and may or may not be running jobs -+ * - We must cause it to leave the runpool by stopping it from -+ * submitting any more jobs. When it finally does leave, -+ * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs -+ * (because it is dying), release the Power Manager active reference, -+ * and will not requeue the context in the queue. -+ * kbase_jd_zap_context() will wait for those jobs to be killed. -+ * - Hence, work required just to make it leave the runpool. Cancelling -+ * jobs and releasing the Power manager active reference will be -+ * handled when it leaves the runpool. -+ */ -+ if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -+ if (!list_empty( -+ &kctx->jctx.sched_info.ctx.ctx_list_entry[js])) -+ list_del_init( -+ &kctx->jctx.sched_info.ctx.ctx_list_entry[js]); -+ } -+ -+ /* The following events require us to kill off remaining jobs -+ * and update PM book-keeping: -+ * - we evicted it correctly (it must have jobs to be in the -+ * Queue) -+ * -+ * These events need no action, but take this path anyway: -+ * - Case a: it didn't have any jobs, and was never in the Queue -+ * - Case b: scheduling transaction will be partially rolled- -+ * back (this already cancels the jobs) -+ */ -+ -+ KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, -+ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); -+ -+ /* Only cancel jobs when we evicted from the -+ * queue. No Power Manager active reference was held. -+ * -+ * Having is_dying set ensures that this kills, and -+ * doesn't requeue */ -+ kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); -+ -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ mutex_unlock(&kctx->jctx.lock); -+ } else { -+ unsigned long flags; -+ bool was_retained; -+ -+ /* Case c: didn't evict, but it is scheduled - it's in the Run -+ * Pool */ -+ KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, -+ kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); -+ -+ /* Disable the ctx from submitting any more jobs */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ kbasep_js_clear_submit_allowed(js_devdata, kctx); -+ -+ /* Retain and (later) release the context whilst it is is now -+ * disallowed from submitting jobs - ensures that someone -+ * somewhere will be removing the context later on */ -+ was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx); -+ -+ /* Since it's scheduled and we have the jsctx_mutex, it must be -+ * retained successfully */ -+ KBASE_DEBUG_ASSERT(was_retained); -+ -+ dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); -+ -+ /* Cancel any remaining running jobs for this kctx - if any. -+ * Submit is disallowed which takes effect immediately, so no -+ * more new jobs will appear after we do this. */ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) -+ kbase_job_slot_hardstop(kctx, js, NULL); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+ mutex_unlock(&js_devdata->queue_mutex); -+ mutex_unlock(&kctx->jctx.lock); -+ -+ dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", -+ kctx); -+ -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ } -+ -+ KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); -+ -+ /* After this, you must wait on both the -+ * kbase_jd_context::zero_jobs_wait and the -+ * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs -+ * to be destroyed, and the context to be de-scheduled (if it was on the -+ * runpool). -+ * -+ * kbase_jd_zap_context() will do this. */ -+} -+ -+static inline int trace_get_refcnt(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ return atomic_read(&kctx->refcount); -+} -+ -+/** -+ * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context -+ * @kctx: Pointer to context. -+ * @callback: Pointer to function to call for each job. -+ * -+ * Call a function on all jobs belonging to a non-queued, non-running -+ * context, and detach the jobs from the context as it goes. -+ * -+ * Due to the locks that might be held at the time of the call, the callback -+ * may need to defer work on a workqueue to complete its actions (e.g. when -+ * cancelling jobs) -+ * -+ * Atoms will be removed from the queue, so this must only be called when -+ * cancelling jobs (which occurs as part of context destruction). -+ * -+ * The locking conditions on the caller are as follows: -+ * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. -+ */ -+static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, -+ kbasep_js_ctx_job_cb callback) -+{ -+ struct kbase_device *kbdev; -+ struct kbasep_js_device_data *js_devdata; -+ unsigned long flags; -+ u32 js; -+ -+ kbdev = kctx->kbdev; -+ js_devdata = &kbdev->js_data; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, -+ 0u, trace_get_refcnt(kbdev, kctx)); -+ -+ /* Invoke callback on jobs on each slot in turn */ -+ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) -+ jsctx_queue_foreach(kctx, js, callback); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.h b/drivers/gpu/arm/midgard/mali_kbase_js.h -new file mode 100755 -index 000000000..ddada8e46 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_js.h -@@ -0,0 +1,925 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_js.h -+ * Job Scheduler APIs. -+ */ -+ -+#ifndef _KBASE_JS_H_ -+#define _KBASE_JS_H_ -+ -+#include "mali_kbase_js_defs.h" -+#include "mali_kbase_context.h" -+#include "mali_kbase_defs.h" -+#include "mali_kbase_debug.h" -+ -+#include "mali_kbase_js_ctx_attr.h" -+ -+/** -+ * @addtogroup base_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup base_kbase_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup kbase_js Job Scheduler Internal APIs -+ * @{ -+ * -+ * These APIs are Internal to KBase. -+ */ -+ -+/** -+ * @brief Initialize the Job Scheduler -+ * -+ * The struct kbasep_js_device_data sub-structure of \a kbdev must be zero -+ * initialized before passing to the kbasep_js_devdata_init() function. This is -+ * to give efficient error path code. -+ */ -+int kbasep_js_devdata_init(struct kbase_device * const kbdev); -+ -+/** -+ * @brief Halt the Job Scheduler. -+ * -+ * It is safe to call this on \a kbdev even if it the kbasep_js_device_data -+ * sub-structure was never initialized/failed initialization, to give efficient -+ * error-path code. -+ * -+ * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must -+ * be zero initialized before passing to the kbasep_js_devdata_init() -+ * function. This is to give efficient error path code. -+ * -+ * It is a Programming Error to call this whilst there are still kbase_context -+ * structures registered with this scheduler. -+ * -+ */ -+void kbasep_js_devdata_halt(struct kbase_device *kbdev); -+ -+/** -+ * @brief Terminate the Job Scheduler -+ * -+ * It is safe to call this on \a kbdev even if it the kbasep_js_device_data -+ * sub-structure was never initialized/failed initialization, to give efficient -+ * error-path code. -+ * -+ * For this to work, the struct kbasep_js_device_data sub-structure of \a kbdev must -+ * be zero initialized before passing to the kbasep_js_devdata_init() -+ * function. This is to give efficient error path code. -+ * -+ * It is a Programming Error to call this whilst there are still kbase_context -+ * structures registered with this scheduler. -+ */ -+void kbasep_js_devdata_term(struct kbase_device *kbdev); -+ -+/** -+ * @brief Initialize the Scheduling Component of a struct kbase_context on the Job Scheduler. -+ * -+ * This effectively registers a struct kbase_context with a Job Scheduler. -+ * -+ * It does not register any jobs owned by the struct kbase_context with the scheduler. -+ * Those must be separately registered by kbasep_js_add_job(). -+ * -+ * The struct kbase_context must be zero intitialized before passing to the -+ * kbase_js_init() function. This is to give efficient error path code. -+ */ -+int kbasep_js_kctx_init(struct kbase_context * const kctx); -+ -+/** -+ * @brief Terminate the Scheduling Component of a struct kbase_context on the Job Scheduler -+ * -+ * This effectively de-registers a struct kbase_context from its Job Scheduler -+ * -+ * It is safe to call this on a struct kbase_context that has never had or failed -+ * initialization of its jctx.sched_info member, to give efficient error-path -+ * code. -+ * -+ * For this to work, the struct kbase_context must be zero intitialized before passing -+ * to the kbase_js_init() function. -+ * -+ * It is a Programming Error to call this whilst there are still jobs -+ * registered with this context. -+ */ -+void kbasep_js_kctx_term(struct kbase_context *kctx); -+ -+/** -+ * @brief Add a job chain to the Job Scheduler, and take necessary actions to -+ * schedule the context/run the job. -+ * -+ * This atomically does the following: -+ * - Update the numbers of jobs information -+ * - Add the job to the run pool if necessary (part of init_job) -+ * -+ * Once this is done, then an appropriate action is taken: -+ * - If the ctx is scheduled, it attempts to start the next job (which might be -+ * this added job) -+ * - Otherwise, and if this is the first job on the context, it enqueues it on -+ * the Policy Queue -+ * -+ * The Policy's Queue can be updated by this in the following ways: -+ * - In the above case that this is the first job on the context -+ * - If the context is high priority and the context is not scheduled, then it -+ * could cause the Policy to schedule out a low-priority context, allowing -+ * this context to be scheduled in. -+ * -+ * If the context is already scheduled on the RunPool, then adding a job to it -+ * is guarenteed not to update the Policy Queue. And so, the caller is -+ * guarenteed to not need to try scheduling a context from the Run Pool - it -+ * can safely assert that the result is false. -+ * -+ * It is a programming error to have more than U32_MAX jobs in flight at a time. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - it must \em not hold hwaccess_lock (as this will be obtained internally) -+ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be -+ * obtained internally) -+ * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). -+ * -+ * @return true indicates that the Policy Queue was updated, and so the -+ * caller will need to try scheduling a context onto the Run Pool. -+ * @return false indicates that no updates were made to the Policy Queue, -+ * so no further action is required from the caller. This is \b always returned -+ * when the context is currently scheduled. -+ */ -+bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); -+ -+/** -+ * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'. -+ * -+ * Completely removing a job requires several calls: -+ * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of -+ * the atom -+ * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler -+ * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the -+ * remaining state held as part of the job having been run. -+ * -+ * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions. -+ * -+ * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions. -+ * -+ * It is a programming error to call this when: -+ * - \a atom is not a job belonging to kctx. -+ * - \a atom has already been removed from the Job Scheduler. -+ * - \a atom is still in the runpool -+ * -+ * Do not use this for removing jobs being killed by kbase_jd_cancel() - use -+ * kbasep_js_remove_cancelled_job() instead. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * -+ */ -+void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *atom); -+ -+/** -+ * @brief Completely remove a job chain from the Job Scheduler, in the case -+ * where the job chain was cancelled. -+ * -+ * This is a variant of kbasep_js_remove_job() that takes care of removing all -+ * of the retained state too. This is generally useful for cancelled atoms, -+ * which need not be handled in an optimal way. -+ * -+ * It is a programming error to call this when: -+ * - \a atom is not a job belonging to kctx. -+ * - \a atom has already been removed from the Job Scheduler. -+ * - \a atom is still in the runpool: -+ * - it is not being killed with kbasep_jd_cancel() -+ * -+ * The following locking conditions are made on the caller: -+ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - it must \em not hold the hwaccess_lock, (as this will be obtained -+ * internally) -+ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be -+ * obtained internally) -+ * -+ * @return true indicates that ctx attributes have changed and the caller -+ * should call kbase_js_sched_all() to try to run more jobs -+ * @return false otherwise -+ */ -+bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, -+ struct kbase_context *kctx, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * @brief Refcount a context as being busy, preventing it from being scheduled -+ * out. -+ * -+ * @note This function can safely be called from IRQ context. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold mmu_hw_mutex and hwaccess_lock, because they will be -+ * used internally. -+ * -+ * @return value != false if the retain succeeded, and the context will not be scheduled out. -+ * @return false if the retain failed (because the context is being/has been scheduled out). -+ */ -+bool kbasep_js_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * @brief Refcount a context as being busy, preventing it from being scheduled -+ * out. -+ * -+ * @note This function can safely be called from IRQ context. -+ * -+ * The following locks must be held by the caller: -+ * - mmu_hw_mutex, hwaccess_lock -+ * -+ * @return value != false if the retain succeeded, and the context will not be scheduled out. -+ * @return false if the retain failed (because the context is being/has been scheduled out). -+ */ -+bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * @brief Lookup a context in the Run Pool based upon its current address space -+ * and ensure that is stays scheduled in. -+ * -+ * The context is refcounted as being busy to prevent it from scheduling -+ * out. It must be released with kbasep_js_runpool_release_ctx() when it is no -+ * longer required to stay scheduled in. -+ * -+ * @note This function can safely be called from IRQ context. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold the hwaccess_lock, because it will be used internally. -+ * If the hwaccess_lock is already held, then the caller should use -+ * kbasep_js_runpool_lookup_ctx_nolock() instead. -+ * -+ * @return a valid struct kbase_context on success, which has been refcounted as being busy. -+ * @return NULL on failure, indicating that no context was found in \a as_nr -+ */ -+struct kbase_context *kbasep_js_runpool_lookup_ctx(struct kbase_device *kbdev, int as_nr); -+ -+/** -+ * @brief Handling the requeuing/killing of a context that was evicted from the -+ * policy queue or runpool. -+ * -+ * This should be used whenever handing off a context that has been evicted -+ * from the policy queue or the runpool: -+ * - If the context is not dying and has jobs, it gets re-added to the policy -+ * queue -+ * - Otherwise, it is not added -+ * -+ * In addition, if the context is dying the jobs are killed asynchronously. -+ * -+ * In all cases, the Power Manager active reference is released -+ * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a -+ * has_pm_ref must be set to false whenever the context was not previously in -+ * the runpool and does not hold a Power Manager active refcount. Note that -+ * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an -+ * active refcount even though they weren't in the runpool. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be -+ * obtained internally) -+ */ -+void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, bool has_pm_ref); -+ -+/** -+ * @brief Release a refcount of a context being busy, allowing it to be -+ * scheduled out. -+ * -+ * When the refcount reaches zero and the context \em might be scheduled out -+ * (depending on whether the Scheudling Policy has deemed it so, or if it has run -+ * out of jobs). -+ * -+ * If the context does get scheduled out, then The following actions will be -+ * taken as part of deschduling a context: -+ * - For the context being descheduled: -+ * - If the context is in the processing of dying (all the jobs are being -+ * removed from it), then descheduling also kills off any jobs remaining in the -+ * context. -+ * - If the context is not dying, and any jobs remain after descheduling the -+ * context then it is re-enqueued to the Policy's Queue. -+ * - Otherwise, the context is still known to the scheduler, but remains absent -+ * from the Policy Queue until a job is next added to it. -+ * - In all descheduling cases, the Power Manager active reference (obtained -+ * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()). -+ * -+ * Whilst the context is being descheduled, this also handles actions that -+ * cause more atoms to be run: -+ * - Attempt submitting atoms when the Context Attributes on the Runpool have -+ * changed. This is because the context being scheduled out could mean that -+ * there are more opportunities to run atoms. -+ * - Attempt submitting to a slot that was previously blocked due to affinity -+ * restrictions. This is usually only necessary when releasing a context -+ * happens as part of completing a previous job, but is harmless nonetheless. -+ * - Attempt scheduling in a new context (if one is available), and if necessary, -+ * running a job from that new context. -+ * -+ * Unlike retaining a context in the runpool, this function \b cannot be called -+ * from IRQ context. -+ * -+ * It is a programming error to call this on a \a kctx that is not currently -+ * scheduled, or that already has a zero refcount. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold the hwaccess_lock, because it will be used internally. -+ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be -+ * obtained internally) -+ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be -+ * obtained internally) -+ * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be -+ * obtained internally) -+ * -+ */ -+void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional -+ * actions from completing an atom. -+ * -+ * This is usually called as part of completing an atom and releasing the -+ * refcount on the context held by the atom. -+ * -+ * Therefore, the extra actions carried out are part of handling actions queued -+ * on a completed atom, namely: -+ * - Releasing the atom's context attributes -+ * - Retrying the submission on a particular slot, because we couldn't submit -+ * on that slot from an IRQ handler. -+ * -+ * The locking conditions of this function are the same as those for -+ * kbasep_js_runpool_release_ctx() -+ */ -+void kbasep_js_runpool_release_ctx_and_katom_retained_state(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); -+ -+/** -+ * @brief Variant of kbase_js_runpool_release_ctx() that assumes that -+ * kbasep_js_device_data::runpool_mutex and -+ * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not -+ * attempt to schedule new contexts. -+ */ -+void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, -+ struct kbase_context *kctx); -+ -+/** -+ * @brief Schedule in a privileged context -+ * -+ * This schedules a context in regardless of the context priority. -+ * If the runpool is full, a context will be forced out of the runpool and the function will wait -+ * for the new context to be scheduled in. -+ * The context will be kept scheduled in (and the corresponding address space reserved) until -+ * kbasep_js_release_privileged_ctx is called). -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold the hwaccess_lock, because it will be used internally. -+ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be -+ * obtained internally) -+ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be -+ * obtained internally) -+ * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally). -+ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will -+ * be used internally. -+ * -+ */ -+void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * @brief Release a privileged context, allowing it to be scheduled out. -+ * -+ * See kbasep_js_runpool_release_ctx for potential side effects. -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold the hwaccess_lock, because it will be used internally. -+ * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be -+ * obtained internally) -+ * - it must \em not hold the kbase_device::mmu_hw_mutex (as this will be -+ * obtained internally) -+ * -+ */ -+void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * @brief Try to submit the next job on each slot -+ * -+ * The following locks may be used: -+ * - kbasep_js_device_data::runpool_mutex -+ * - hwaccess_lock -+ */ -+void kbase_js_try_run_jobs(struct kbase_device *kbdev); -+ -+/** -+ * @brief Suspend the job scheduler during a Power Management Suspend event. -+ * -+ * Causes all contexts to be removed from the runpool, and prevents any -+ * contexts from (re)entering the runpool. -+ * -+ * This does not handle suspending the one privileged context: the caller must -+ * instead do this by by suspending the GPU HW Counter Instrumentation. -+ * -+ * This will eventually cause all Power Management active references held by -+ * contexts on the runpool to be released, without running any more atoms. -+ * -+ * The caller must then wait for all Power Mangement active refcount to become -+ * zero before completing the suspend. -+ * -+ * The emptying mechanism may take some time to complete, since it can wait for -+ * jobs to complete naturally instead of forcing them to end quickly. However, -+ * this is bounded by the Job Scheduler's Job Timeouts. Hence, this -+ * function is guaranteed to complete in a finite time. -+ */ -+void kbasep_js_suspend(struct kbase_device *kbdev); -+ -+/** -+ * @brief Resume the Job Scheduler after a Power Management Resume event. -+ * -+ * This restores the actions from kbasep_js_suspend(): -+ * - Schedules contexts back into the runpool -+ * - Resumes running atoms on the GPU -+ */ -+void kbasep_js_resume(struct kbase_device *kbdev); -+ -+/** -+ * @brief Submit an atom to the job scheduler. -+ * -+ * The atom is enqueued on the context's ringbuffer. The caller must have -+ * ensured that all dependencies can be represented in the ringbuffer. -+ * -+ * Caller must hold jctx->lock -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] atom Pointer to the atom to submit -+ * -+ * @return Whether the context requires to be enqueued. */ -+bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer. -+ * @kctx: Context Pointer -+ * @prio: Priority (specifies the queue together with js). -+ * @js: Job slot (specifies the queue together with prio). -+ * -+ * Pushes all possible atoms from the linked list to the ringbuffer. -+ * Number of atoms are limited to free space in the ringbuffer and -+ * number of available atoms in the linked list. -+ * -+ */ -+void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js); -+/** -+ * @brief Pull an atom from a context in the job scheduler for execution. -+ * -+ * The atom will not be removed from the ringbuffer at this stage. -+ * -+ * The HW access lock must be held when calling this function. -+ * -+ * @param[in] kctx Context to pull from -+ * @param[in] js Job slot to pull from -+ * @return Pointer to an atom, or NULL if there are no atoms for this -+ * slot that can be currently run. -+ */ -+struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); -+ -+/** -+ * @brief Return an atom to the job scheduler ringbuffer. -+ * -+ * An atom is 'unpulled' if execution is stopped but intended to be returned to -+ * later. The most common reason for this is that the atom has been -+ * soft-stopped. -+ * -+ * Note that if multiple atoms are to be 'unpulled', they must be returned in -+ * the reverse order to which they were originally pulled. It is a programming -+ * error to return atoms in any other order. -+ * -+ * The HW access lock must be held when calling this function. -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] atom Pointer to the atom to unpull -+ */ -+void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); -+ -+/** -+ * @brief Complete an atom from jd_done_worker(), removing it from the job -+ * scheduler ringbuffer. -+ * -+ * If the atom failed then all dependee atoms marked for failure propagation -+ * will also fail. -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] katom Pointer to the atom to complete -+ * @return true if the context is now idle (no jobs pulled) -+ * false otherwise -+ */ -+bool kbase_js_complete_atom_wq(struct kbase_context *kctx, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * @brief Complete an atom. -+ * -+ * Most of the work required to complete an atom will be performed by -+ * jd_done_worker(). -+ * -+ * The HW access lock must be held when calling this function. -+ * -+ * @param[in] katom Pointer to the atom to complete -+ * @param[in] end_timestamp The time that the atom completed (may be NULL) -+ * -+ * Return: Atom that has now been unblocked and can now be run, or NULL if none -+ */ -+struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, -+ ktime_t *end_timestamp); -+ -+/** -+ * @brief Submit atoms from all available contexts. -+ * -+ * This will attempt to submit as many jobs as possible to the provided job -+ * slots. It will exit when either all job slots are full, or all contexts have -+ * been used. -+ * -+ * @param[in] kbdev Device pointer -+ * @param[in] js_mask Mask of job slots to submit to -+ */ -+void kbase_js_sched(struct kbase_device *kbdev, int js_mask); -+ -+/** -+ * kbase_jd_zap_context - Attempt to deschedule a context that is being -+ * destroyed -+ * @kctx: Context pointer -+ * -+ * This will attempt to remove a context from any internal job scheduler queues -+ * and perform any other actions to ensure a context will not be submitted -+ * from. -+ * -+ * If the context is currently scheduled, then the caller must wait for all -+ * pending jobs to complete before taking any further action. -+ */ -+void kbase_js_zap_context(struct kbase_context *kctx); -+ -+/** -+ * @brief Validate an atom -+ * -+ * This will determine whether the atom can be scheduled onto the GPU. Atoms -+ * with invalid combinations of core requirements will be rejected. -+ * -+ * @param[in] kbdev Device pointer -+ * @param[in] katom Atom to validate -+ * @return true if atom is valid -+ * false otherwise -+ */ -+bool kbase_js_is_atom_valid(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_js_set_timeouts - update all JS timeouts with user specified data -+ * @kbdev: Device pointer -+ * -+ * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is -+ * set to a positive number then that becomes the new value used, if a timeout -+ * is negative then the default is set. -+ */ -+void kbase_js_set_timeouts(struct kbase_device *kbdev); -+ -+/* -+ * Helpers follow -+ */ -+ -+/** -+ * @brief Check that a context is allowed to submit jobs on this policy -+ * -+ * The purpose of this abstraction is to hide the underlying data size, and wrap up -+ * the long repeated line of code. -+ * -+ * As with any bool, never test the return value with true. -+ * -+ * The caller must hold hwaccess_lock. -+ */ -+static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) -+{ -+ u16 test_bit; -+ -+ /* Ensure context really is scheduled in */ -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ test_bit = (u16) (1u << kctx->as_nr); -+ -+ return (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); -+} -+ -+/** -+ * @brief Allow a context to submit jobs on this policy -+ * -+ * The purpose of this abstraction is to hide the underlying data size, and wrap up -+ * the long repeated line of code. -+ * -+ * The caller must hold hwaccess_lock. -+ */ -+static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) -+{ -+ u16 set_bit; -+ -+ /* Ensure context really is scheduled in */ -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ set_bit = (u16) (1u << kctx->as_nr); -+ -+ dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); -+ -+ js_devdata->runpool_irq.submit_allowed |= set_bit; -+} -+ -+/** -+ * @brief Prevent a context from submitting more jobs on this policy -+ * -+ * The purpose of this abstraction is to hide the underlying data size, and wrap up -+ * the long repeated line of code. -+ * -+ * The caller must hold hwaccess_lock. -+ */ -+static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, struct kbase_context *kctx) -+{ -+ u16 clear_bit; -+ u16 clear_mask; -+ -+ /* Ensure context really is scheduled in */ -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ clear_bit = (u16) (1u << kctx->as_nr); -+ clear_mask = ~clear_bit; -+ -+ dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr); -+ -+ js_devdata->runpool_irq.submit_allowed &= clear_mask; -+} -+ -+/** -+ * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom -+ */ -+static inline void kbasep_js_clear_job_retry_submit(struct kbase_jd_atom *atom) -+{ -+ atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; -+} -+ -+/** -+ * Mark a slot as requiring resubmission by carrying that information on a -+ * completing atom. -+ * -+ * @note This can ASSERT in debug builds if the submit slot has been set to -+ * something other than the current value for @a js. This is because you might -+ * be unintentionally stopping more jobs being submitted on the old submit -+ * slot, and that might cause a scheduling-hang. -+ * -+ * @note If you can guarantee that the atoms for the original slot will be -+ * submitted on some other slot, then call kbasep_js_clear_job_retry_submit() -+ * first to silence the ASSERT. -+ */ -+static inline void kbasep_js_set_job_retry_submit_slot(struct kbase_jd_atom *atom, int js) -+{ -+ KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS); -+ KBASE_DEBUG_ASSERT((atom->retry_submit_on_slot == -+ KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID) -+ || (atom->retry_submit_on_slot == js)); -+ -+ atom->retry_submit_on_slot = js; -+} -+ -+/** -+ * Create an initial 'invalid' atom retained state, that requires no -+ * atom-related work to be done on releasing with -+ * kbasep_js_runpool_release_ctx_and_katom_retained_state() -+ */ -+static inline void kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) -+{ -+ retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; -+ retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; -+ retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID; -+} -+ -+/** -+ * Copy atom state that can be made available after jd_done_nolock() is called -+ * on that atom. -+ */ -+static inline void kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, const struct kbase_jd_atom *katom) -+{ -+ retained_state->event_code = katom->event_code; -+ retained_state->core_req = katom->core_req; -+ retained_state->retry_submit_on_slot = katom->retry_submit_on_slot; -+ retained_state->sched_priority = katom->sched_priority; -+ retained_state->device_nr = katom->device_nr; -+} -+ -+/** -+ * @brief Determine whether an atom has finished (given its retained state), -+ * and so should be given back to userspace/removed from the system. -+ * -+ * Reasons for an atom not finishing include: -+ * - Being soft-stopped (and so, the atom should be resubmitted sometime later) -+ * -+ * @param[in] katom_retained_state the retained state of the atom to check -+ * @return false if the atom has not finished -+ * @return !=false if the atom has finished -+ */ -+static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) -+{ -+ return (bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); -+} -+ -+/** -+ * @brief Determine whether a struct kbasep_js_atom_retained_state is valid -+ * -+ * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates that the -+ * code should just ignore it. -+ * -+ * @param[in] katom_retained_state the atom's retained state to check -+ * @return false if the retained state is invalid, and can be ignored -+ * @return !=false if the retained state is valid -+ */ -+static inline bool kbasep_js_atom_retained_state_is_valid(const struct kbasep_js_atom_retained_state *katom_retained_state) -+{ -+ return (bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); -+} -+ -+static inline bool kbasep_js_get_atom_retry_submit_slot(const struct kbasep_js_atom_retained_state *katom_retained_state, int *res) -+{ -+ int js = katom_retained_state->retry_submit_on_slot; -+ -+ *res = js; -+ return (bool) (js >= 0); -+} -+ -+/** -+ * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the -+ * context is guaranteed to be already previously retained. -+ * -+ * It is a programming error to supply the \a as_nr of a context that has not -+ * been previously retained/has a busy refcount of zero. The only exception is -+ * when there is no ctx in \a as_nr (NULL returned). -+ * -+ * The following locking conditions are made on the caller: -+ * - it must \em not hold the hwaccess_lock, because it will be used internally. -+ * -+ * @return a valid struct kbase_context on success, with a refcount that is guaranteed -+ * to be non-zero and unmodified by this function. -+ * @return NULL on failure, indicating that no context was found in \a as_nr -+ */ -+static inline struct kbase_context *kbasep_js_runpool_lookup_ctx_noretain(struct kbase_device *kbdev, int as_nr) -+{ -+ struct kbase_context *found_kctx; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS); -+ -+ found_kctx = kbdev->as_to_kctx[as_nr]; -+ KBASE_DEBUG_ASSERT(found_kctx == NULL || -+ atomic_read(&found_kctx->refcount) > 0); -+ -+ return found_kctx; -+} -+ -+/* -+ * The following locking conditions are made on the caller: -+ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - The caller must hold the kbasep_js_device_data::runpool_mutex -+ */ -+static inline void kbase_js_runpool_inc_context_count( -+ struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ -+ /* Track total contexts */ -+ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX); -+ ++(js_devdata->nr_all_contexts_running); -+ -+ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { -+ /* Track contexts that can submit jobs */ -+ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running < -+ S8_MAX); -+ ++(js_devdata->nr_user_contexts_running); -+ } -+} -+ -+/* -+ * The following locking conditions are made on the caller: -+ * - The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. -+ * - The caller must hold the kbasep_js_device_data::runpool_mutex -+ */ -+static inline void kbase_js_runpool_dec_context_count( -+ struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ lockdep_assert_held(&js_devdata->runpool_mutex); -+ -+ /* Track total contexts */ -+ --(js_devdata->nr_all_contexts_running); -+ KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0); -+ -+ if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { -+ /* Track contexts that can submit jobs */ -+ --(js_devdata->nr_user_contexts_running); -+ KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0); -+ } -+} -+ -+ -+/** -+ * @brief Submit atoms from all available contexts to all job slots. -+ * -+ * This will attempt to submit as many jobs as possible. It will exit when -+ * either all job slots are full, or all contexts have been used. -+ * -+ * @param[in] kbdev Device pointer -+ */ -+static inline void kbase_js_sched_all(struct kbase_device *kbdev) -+{ -+ kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); -+} -+ -+extern const int -+kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; -+ -+extern const base_jd_prio -+kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -+ -+/** -+ * kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio) -+ * to relative ordering -+ * @atom_prio: Priority ID to translate. -+ * -+ * Atom priority values for @ref base_jd_prio cannot be compared directly to -+ * find out which are higher or lower. -+ * -+ * This function will convert base_jd_prio values for successively lower -+ * priorities into a monotonically increasing sequence. That is, the lower the -+ * base_jd_prio priority, the higher the value produced by this function. This -+ * is in accordance with how the rest of the kernel treates priority. -+ * -+ * The mapping is 1:1 and the size of the valid input range is the same as the -+ * size of the valid output range, i.e. -+ * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS -+ * -+ * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions -+ * -+ * Return: On success: a value in the inclusive range -+ * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: -+ * KBASE_JS_ATOM_SCHED_PRIO_INVALID -+ */ -+static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) -+{ -+ if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) -+ return KBASE_JS_ATOM_SCHED_PRIO_INVALID; -+ -+ return kbasep_js_atom_priority_to_relative[atom_prio]; -+} -+ -+static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) -+{ -+ unsigned int prio_idx; -+ -+ KBASE_DEBUG_ASSERT(0 <= sched_prio -+ && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT); -+ -+ prio_idx = (unsigned int)sched_prio; -+ -+ return kbasep_js_relative_priority_to_atom[prio_idx]; -+} -+ -+ /** @} *//* end group kbase_js */ -+ /** @} *//* end group base_kbase_api */ -+ /** @} *//* end group base_api */ -+ -+#endif /* _KBASE_JS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c -new file mode 100755 -index 000000000..321506ada ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c -@@ -0,0 +1,301 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+#include -+#include -+ -+/* -+ * Private functions follow -+ */ -+ -+/** -+ * @brief Check whether a ctx has a certain attribute, and if so, retain that -+ * attribute on the runpool. -+ * -+ * Requires: -+ * - jsctx mutex -+ * - runpool_irq spinlock -+ * - ctx is scheduled on the runpool -+ * -+ * @return true indicates a change in ctx attributes state of the runpool. -+ * In this state, the scheduler might be able to submit more jobs than -+ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() -+ * or similar is called sometime later. -+ * @return false indicates no change in ctx attributes state of the runpool. -+ */ -+static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ bool runpool_state_changed = false; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { -+ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); -+ ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); -+ -+ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { -+ /* First refcount indicates a state change */ -+ runpool_state_changed = true; -+ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute); -+ } -+ } -+ -+ return runpool_state_changed; -+} -+ -+/** -+ * @brief Check whether a ctx has a certain attribute, and if so, release that -+ * attribute on the runpool. -+ * -+ * Requires: -+ * - jsctx mutex -+ * - runpool_irq spinlock -+ * - ctx is scheduled on the runpool -+ * -+ * @return true indicates a change in ctx attributes state of the runpool. -+ * In this state, the scheduler might be able to submit more jobs than -+ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() -+ * or similar is called sometime later. -+ * @return false indicates no change in ctx attributes state of the runpool. -+ */ -+static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ struct kbasep_js_kctx_info *js_kctx_info; -+ bool runpool_state_changed = false; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); -+ js_devdata = &kbdev->js_data; -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -+ -+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { -+ KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); -+ --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); -+ -+ if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { -+ /* Last de-refcount indicates a state change */ -+ runpool_state_changed = true; -+ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute); -+ } -+ } -+ -+ return runpool_state_changed; -+} -+ -+/** -+ * @brief Retain a certain attribute on a ctx, also retaining it on the runpool -+ * if the context is scheduled. -+ * -+ * Requires: -+ * - jsctx mutex -+ * - If the context is scheduled, then runpool_irq spinlock must also be held -+ * -+ * @return true indicates a change in ctx attributes state of the runpool. -+ * This may allow the scheduler to submit more jobs than previously. -+ * @return false indicates no change in ctx attributes state of the runpool. -+ */ -+static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ bool runpool_state_changed = false; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); -+ -+ ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); -+ -+ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { -+ /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ -+ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); -+ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); -+ } -+ -+ return runpool_state_changed; -+} -+ -+/* -+ * @brief Release a certain attribute on a ctx, also releasing it from the runpool -+ * if the context is scheduled. -+ * -+ * Requires: -+ * - jsctx mutex -+ * - If the context is scheduled, then runpool_irq spinlock must also be held -+ * -+ * @return true indicates a change in ctx attributes state of the runpool. -+ * This may allow the scheduler to submit more jobs than previously. -+ * @return false indicates no change in ctx attributes state of the runpool. -+ */ -+static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ bool runpool_state_changed = false; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); -+ KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); -+ -+ if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ -+ runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); -+ KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); -+ } -+ -+ /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ -+ --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); -+ -+ return runpool_state_changed; -+} -+ -+/* -+ * More commonly used public functions -+ */ -+ -+void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx) -+{ -+ bool runpool_state_changed = false; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { -+ /* This context never submits, so don't track any scheduling attributes */ -+ return; -+ } -+ -+ /* Transfer attributes held in the context flags for contexts that have submit enabled */ -+ -+ /* ... More attributes can be added here ... */ -+ -+ /* The context should not have been scheduled yet, so ASSERT if this caused -+ * runpool state changes (note that other threads *can't* affect the value -+ * of runpool_state_changed, due to how it's calculated) */ -+ KBASE_DEBUG_ASSERT(runpool_state_changed == false); -+ CSTD_UNUSED(runpool_state_changed); -+} -+ -+void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -+{ -+ bool runpool_state_changed; -+ int i; -+ -+ /* Retain any existing attributes */ -+ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { -+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { -+ /* The context is being scheduled in, so update the runpool with the new attributes */ -+ runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); -+ -+ /* We don't need to know about state changed, because retaining a -+ * context occurs on scheduling it, and that itself will also try -+ * to run new atoms */ -+ CSTD_UNUSED(runpool_state_changed); -+ } -+ } -+} -+ -+bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -+{ -+ bool runpool_state_changed = false; -+ int i; -+ -+ /* Release any existing attributes */ -+ for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { -+ if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) { -+ /* The context is being scheduled out, so update the runpool on the removed attributes */ -+ runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i); -+ } -+ } -+ -+ return runpool_state_changed; -+} -+ -+void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ bool runpool_state_changed = false; -+ base_jd_core_req core_req; -+ -+ KBASE_DEBUG_ASSERT(katom); -+ core_req = katom->core_req; -+ -+ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) -+ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); -+ else -+ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); -+ -+ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { -+ /* Atom that can run on slot1 or slot2, and can use all cores */ -+ runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); -+ } -+ -+ /* We don't need to know about state changed, because retaining an -+ * atom occurs on adding it, and that itself will also try to run -+ * new atoms */ -+ CSTD_UNUSED(runpool_state_changed); -+} -+ -+bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state) -+{ -+ bool runpool_state_changed = false; -+ base_jd_core_req core_req; -+ -+ KBASE_DEBUG_ASSERT(katom_retained_state); -+ core_req = katom_retained_state->core_req; -+ -+ /* No-op for invalid atoms */ -+ if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) -+ return false; -+ -+ if (core_req & BASE_JD_REQ_ONLY_COMPUTE) -+ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); -+ else -+ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); -+ -+ if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { -+ /* Atom that can run on slot1 or slot2, and can use all cores */ -+ runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); -+ } -+ -+ return runpool_state_changed; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h -new file mode 100755 -index 000000000..ce9183326 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h -@@ -0,0 +1,158 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_js_ctx_attr.h -+ * Job Scheduler Context Attribute APIs -+ */ -+ -+#ifndef _KBASE_JS_CTX_ATTR_H_ -+#define _KBASE_JS_CTX_ATTR_H_ -+ -+/** -+ * @addtogroup base_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup base_kbase_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup kbase_js -+ * @{ -+ */ -+ -+/** -+ * Set the initial attributes of a context (when context create flags are set) -+ * -+ * Requires: -+ * - Hold the jsctx_mutex -+ */ -+void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * Retain all attributes of a context -+ * -+ * This occurs on scheduling in the context on the runpool (but after -+ * is_scheduled is set) -+ * -+ * Requires: -+ * - jsctx mutex -+ * - runpool_irq spinlock -+ * - ctx->is_scheduled is true -+ */ -+void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * Release all attributes of a context -+ * -+ * This occurs on scheduling out the context from the runpool (but before -+ * is_scheduled is cleared) -+ * -+ * Requires: -+ * - jsctx mutex -+ * - runpool_irq spinlock -+ * - ctx->is_scheduled is true -+ * -+ * @return true indicates a change in ctx attributes state of the runpool. -+ * In this state, the scheduler might be able to submit more jobs than -+ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() -+ * or similar is called sometime later. -+ * @return false indicates no change in ctx attributes state of the runpool. -+ */ -+bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); -+ -+/** -+ * Retain all attributes of an atom -+ * -+ * This occurs on adding an atom to a context -+ * -+ * Requires: -+ * - jsctx mutex -+ * - If the context is scheduled, then runpool_irq spinlock must also be held -+ */ -+void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); -+ -+/** -+ * Release all attributes of an atom, given its retained state. -+ * -+ * This occurs after (permanently) removing an atom from a context -+ * -+ * Requires: -+ * - jsctx mutex -+ * - If the context is scheduled, then runpool_irq spinlock must also be held -+ * -+ * This is a no-op when \a katom_retained_state is invalid. -+ * -+ * @return true indicates a change in ctx attributes state of the runpool. -+ * In this state, the scheduler might be able to submit more jobs than -+ * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() -+ * or similar is called sometime later. -+ * @return false indicates no change in ctx attributes state of the runpool. -+ */ -+bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); -+ -+/** -+ * Requires: -+ * - runpool_irq spinlock -+ */ -+static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) -+{ -+ struct kbasep_js_device_data *js_devdata; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); -+ js_devdata = &kbdev->js_data; -+ -+ return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; -+} -+ -+/** -+ * Requires: -+ * - runpool_irq spinlock -+ */ -+static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, enum kbasep_js_ctx_attr attribute) -+{ -+ /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ -+ return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); -+} -+ -+/** -+ * Requires: -+ * - jsctx mutex -+ */ -+static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute) -+{ -+ struct kbasep_js_kctx_info *js_kctx_info; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); -+ js_kctx_info = &kctx->jctx.sched_info; -+ -+ /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ -+ return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); -+} -+ -+ /** @} *//* end group kbase_js */ -+ /** @} *//* end group base_kbase_api */ -+ /** @} *//* end group base_api */ -+ -+#endif /* _KBASE_JS_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_js_defs.h b/drivers/gpu/arm/midgard/mali_kbase_js_defs.h -new file mode 100755 -index 000000000..ba8b64415 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_js_defs.h -@@ -0,0 +1,386 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_js.h -+ * Job Scheduler Type Definitions -+ */ -+ -+#ifndef _KBASE_JS_DEFS_H_ -+#define _KBASE_JS_DEFS_H_ -+ -+/** -+ * @addtogroup base_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup base_kbase_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup kbase_js -+ * @{ -+ */ -+/* Forward decls */ -+struct kbase_device; -+struct kbase_jd_atom; -+ -+ -+typedef u32 kbase_context_flags; -+ -+struct kbasep_atom_req { -+ base_jd_core_req core_req; -+ kbase_context_flags ctx_req; -+ u32 device_nr; -+}; -+ -+/** Callback function run on all of a context's jobs registered with the Job -+ * Scheduler */ -+typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); -+ -+/** -+ * @brief Maximum number of jobs that can be submitted to a job slot whilst -+ * inside the IRQ handler. -+ * -+ * This is important because GPU NULL jobs can complete whilst the IRQ handler -+ * is running. Otherwise, it potentially allows an unlimited number of GPU NULL -+ * jobs to be submitted inside the IRQ handler, which increases IRQ latency. -+ */ -+#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 -+ -+/** -+ * @brief Context attributes -+ * -+ * Each context attribute can be thought of as a boolean value that caches some -+ * state information about either the runpool, or the context: -+ * - In the case of the runpool, it is a cache of "Do any contexts owned by -+ * the runpool have attribute X?" -+ * - In the case of a context, it is a cache of "Do any atoms owned by the -+ * context have attribute X?" -+ * -+ * The boolean value of the context attributes often affect scheduling -+ * decisions, such as affinities to use and job slots to use. -+ * -+ * To accomodate changes of state in the context, each attribute is refcounted -+ * in the context, and in the runpool for all running contexts. Specifically: -+ * - The runpool holds a refcount of how many contexts in the runpool have this -+ * attribute. -+ * - The context holds a refcount of how many atoms have this attribute. -+ */ -+enum kbasep_js_ctx_attr { -+ /** Attribute indicating a context that contains Compute jobs. That is, -+ * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE -+ * -+ * @note A context can be both 'Compute' and 'Non Compute' if it contains -+ * both types of jobs. -+ */ -+ KBASEP_JS_CTX_ATTR_COMPUTE, -+ -+ /** Attribute indicating a context that contains Non-Compute jobs. That is, -+ * the context has some jobs that are \b not of type @ref -+ * BASE_JD_REQ_ONLY_COMPUTE. -+ * -+ * @note A context can be both 'Compute' and 'Non Compute' if it contains -+ * both types of jobs. -+ */ -+ KBASEP_JS_CTX_ATTR_NON_COMPUTE, -+ -+ /** Attribute indicating that a context contains compute-job atoms that -+ * aren't restricted to a coherent group, and can run on all cores. -+ * -+ * Specifically, this is when the atom's \a core_req satisfy: -+ * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 -+ * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups -+ * -+ * Such atoms could be blocked from running if one of the coherent groups -+ * is being used by another job slot, so tracking this context attribute -+ * allows us to prevent such situations. -+ * -+ * @note This doesn't take into account the 1-coregroup case, where all -+ * compute atoms would effectively be able to run on 'all cores', but -+ * contexts will still not always get marked with this attribute. Instead, -+ * it is the caller's responsibility to take into account the number of -+ * coregroups when interpreting this attribute. -+ * -+ * @note Whilst Tiler atoms are normally combined with -+ * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without -+ * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy -+ * enough to handle anyway. -+ */ -+ KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, -+ -+ /** Must be the last in the enum */ -+ KBASEP_JS_CTX_ATTR_COUNT -+}; -+ -+enum { -+ /** Bit indicating that new atom should be started because this atom completed */ -+ KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), -+ /** Bit indicating that the atom was evicted from the JS_NEXT registers */ -+ KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) -+}; -+ -+/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ -+typedef u32 kbasep_js_atom_done_code; -+ -+/** -+ * @brief KBase Device Data Job Scheduler sub-structure -+ * -+ * This encapsulates the current context of the Job Scheduler on a particular -+ * device. This context is global to the device, and is not tied to any -+ * particular struct kbase_context running on the device. -+ * -+ * nr_contexts_running and as_free are optimized for packing together (by making -+ * them smaller types than u32). The operations on them should rarely involve -+ * masking. The use of signed types for arithmetic indicates to the compiler that -+ * the value will not rollover (which would be undefined behavior), and so under -+ * the Total License model, it is free to make optimizations based on that (i.e. -+ * to remove masking). -+ */ -+struct kbasep_js_device_data { -+ /* Sub-structure to collect together Job Scheduling data used in IRQ -+ * context. The hwaccess_lock must be held when accessing. */ -+ struct runpool_irq { -+ /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. -+ * When bit 'N' is set in this, it indicates whether the context bound to address space -+ * 'N' is allowed to submit jobs. -+ */ -+ u16 submit_allowed; -+ -+ /** Context Attributes: -+ * Each is large enough to hold a refcount of the number of contexts -+ * that can fit into the runpool. This is currently BASE_MAX_NR_AS -+ * -+ * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store -+ * the refcount. Hence, it's not worthwhile reducing this to -+ * bit-manipulation on u32s to save space (where in contrast, 4 bit -+ * sub-fields would be easy to do and would save space). -+ * -+ * Whilst this must not become negative, the sign bit is used for: -+ * - error detection in debug builds -+ * - Optimization: it is undefined for a signed int to overflow, and so -+ * the compiler can optimize for that never happening (thus, no masking -+ * is required on updating the variable) */ -+ s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; -+ -+ /* -+ * Affinity management and tracking -+ */ -+ /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates -+ * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ -+ u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; -+ /** Refcount for each core owned by each slot. Used to generate the -+ * slot_affinities array of bitvectors -+ * -+ * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, -+ * because it is refcounted only when a job is definitely about to be -+ * submitted to a slot, and is de-refcounted immediately after a job -+ * finishes */ -+ s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; -+ } runpool_irq; -+ -+ /** -+ * Run Pool mutex, for managing contexts within the runpool. -+ * Unless otherwise specified, you must hold this lock whilst accessing any -+ * members that follow -+ * -+ * In addition, this is used to access: -+ * - the kbasep_js_kctx_info::runpool substructure -+ */ -+ struct mutex runpool_mutex; -+ -+ /** -+ * Queue Lock, used to access the Policy's queue of contexts independently -+ * of the Run Pool. -+ * -+ * Of course, you don't need the Run Pool lock to access this. -+ */ -+ struct mutex queue_mutex; -+ -+ /** -+ * Scheduling semaphore. This must be held when calling -+ * kbase_jm_kick() -+ */ -+ struct semaphore schedule_sem; -+ -+ /** -+ * List of contexts that can currently be pulled from -+ */ -+ struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS]; -+ /** -+ * List of contexts that can not currently be pulled from, but have -+ * jobs currently running. -+ */ -+ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS]; -+ -+ /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ -+ s8 nr_user_contexts_running; -+ /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ -+ s8 nr_all_contexts_running; -+ -+ /** Core Requirements to match up with base_js_atom's core_req memeber -+ * @note This is a write-once member, and so no locking is required to read */ -+ base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; -+ -+ u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ -+ u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ -+ u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ -+ u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ -+ u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ -+ u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ -+ u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ -+ u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ -+ u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ -+ u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ -+ -+ /**< Value for JS_SOFT_JOB_TIMEOUT */ -+ atomic_t soft_job_timeout_ms; -+ -+ /** List of suspended soft jobs */ -+ struct list_head suspended_soft_jobs_list; -+ -+#ifdef CONFIG_MALI_DEBUG -+ /* Support soft-stop on a single context */ -+ bool softstop_always; -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ /** The initalized-flag is placed at the end, to avoid cache-pollution (we should -+ * only be using this during init/term paths). -+ * @note This is a write-once member, and so no locking is required to read */ -+ int init_status; -+ -+ /* Number of contexts that can currently be pulled from */ -+ u32 nr_contexts_pullable; -+ -+ /* Number of contexts that can either be pulled from or are currently -+ * running */ -+ atomic_t nr_contexts_runnable; -+}; -+ -+/** -+ * @brief KBase Context Job Scheduling information structure -+ * -+ * This is a substructure in the struct kbase_context that encapsulates all the -+ * scheduling information. -+ */ -+struct kbasep_js_kctx_info { -+ -+ /** -+ * Job Scheduler Context information sub-structure. These members are -+ * accessed regardless of whether the context is: -+ * - In the Policy's Run Pool -+ * - In the Policy's Queue -+ * - Not queued nor in the Run Pool. -+ * -+ * You must obtain the jsctx_mutex before accessing any other members of -+ * this substructure. -+ * -+ * You may not access any of these members from IRQ context. -+ */ -+ struct kbase_jsctx { -+ struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ -+ -+ /** Number of jobs ready to run - does \em not include the jobs waiting in -+ * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr -+ * for such jobs*/ -+ u32 nr_jobs; -+ -+ /** Context Attributes: -+ * Each is large enough to hold a refcount of the number of atoms on -+ * the context. **/ -+ u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; -+ -+ /** -+ * Wait queue to wait for KCTX_SHEDULED flag state changes. -+ * */ -+ wait_queue_head_t is_scheduled_wait; -+ -+ /** Link implementing JS queues. Context can be present on one -+ * list per job slot -+ */ -+ struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; -+ } ctx; -+ -+ /* The initalized-flag is placed at the end, to avoid cache-pollution (we should -+ * only be using this during init/term paths) */ -+ int init_status; -+}; -+ -+/** Subset of atom state that can be available after jd_done_nolock() is called -+ * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), -+ * because the original atom could disappear. */ -+struct kbasep_js_atom_retained_state { -+ /** Event code - to determine whether the atom has finished */ -+ enum base_jd_event_code event_code; -+ /** core requirements */ -+ base_jd_core_req core_req; -+ /* priority */ -+ int sched_priority; -+ /** Job Slot to retry submitting to if submission from IRQ handler failed */ -+ int retry_submit_on_slot; -+ /* Core group atom was executed on */ -+ u32 device_nr; -+ -+}; -+ -+/** -+ * Value signifying 'no retry on a slot required' for: -+ * - kbase_js_atom_retained_state::retry_submit_on_slot -+ * - kbase_jd_atom::retry_submit_on_slot -+ */ -+#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) -+ -+/** -+ * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. -+ * -+ * @see kbase_atom_retained_state_is_valid() -+ */ -+#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP -+ -+/** -+ * @brief The JS timer resolution, in microseconds -+ * -+ * Any non-zero difference in time will be at least this size. -+ */ -+#define KBASEP_JS_TICK_RESOLUTION_US 1 -+ -+/* -+ * Internal atom priority defines for kbase_jd_atom::sched_prio -+ */ -+enum { -+ KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, -+ KBASE_JS_ATOM_SCHED_PRIO_MED, -+ KBASE_JS_ATOM_SCHED_PRIO_LOW, -+ KBASE_JS_ATOM_SCHED_PRIO_COUNT, -+}; -+ -+/* Invalid priority for kbase_jd_atom::sched_prio */ -+#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 -+ -+/* Default priority in the case of contexts with no atoms, or being lenient -+ * about invalid priorities from userspace */ -+#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED -+ -+ /** @} *//* end group kbase_js */ -+ /** @} *//* end group base_kbase_api */ -+ /** @} *//* end group base_api */ -+ -+#endif /* _KBASE_JS_DEFS_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_linux.h b/drivers/gpu/arm/midgard/mali_kbase_linux.h -new file mode 100755 -index 000000000..6d1e61fd4 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_linux.h -@@ -0,0 +1,43 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_linux.h -+ * Base kernel APIs, Linux implementation. -+ */ -+ -+#ifndef _KBASE_LINUX_H_ -+#define _KBASE_LINUX_H_ -+ -+/* All things that are needed for the Linux port. */ -+#include -+#include -+#include -+#include -+#include -+ -+#if (defined(MALI_KERNEL_TEST_API) && (1 == MALI_KERNEL_TEST_API)) -+ #define KBASE_EXPORT_TEST_API(func) EXPORT_SYMBOL(func) -+#else -+ #define KBASE_EXPORT_TEST_API(func) -+#endif -+ -+#define KBASE_EXPORT_SYMBOL(func) EXPORT_SYMBOL(func) -+ -+#endif /* _KBASE_LINUX_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem.c b/drivers/gpu/arm/midgard/mali_kbase_mem.c -new file mode 100755 -index 000000000..8cbb5f676 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem.c -@@ -0,0 +1,2657 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_mem.c -+ * Base kernel memory APIs -+ */ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+#include -+#endif /* CONFIG_DMA_SHARED_BUFFER */ -+#ifdef CONFIG_UMP -+#include -+#endif /* CONFIG_UMP */ -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/* This function finds out which RB tree the given GPU VA region belongs to -+ * based on the region zone */ -+static struct rb_root *kbase_reg_flags_to_rbtree(struct kbase_context *kctx, -+ struct kbase_va_region *reg) -+{ -+ struct rb_root *rbtree = NULL; -+ -+ switch (reg->flags & KBASE_REG_ZONE_MASK) { -+ case KBASE_REG_ZONE_CUSTOM_VA: -+ rbtree = &kctx->reg_rbtree_custom; -+ break; -+ case KBASE_REG_ZONE_EXEC: -+ rbtree = &kctx->reg_rbtree_exec; -+ break; -+ case KBASE_REG_ZONE_SAME_VA: -+ rbtree = &kctx->reg_rbtree_same; -+ /* fall through */ -+ default: -+ rbtree = &kctx->reg_rbtree_same; -+ break; -+ } -+ -+ return rbtree; -+} -+ -+/* This function finds out which RB tree the given pfn from the GPU VA belongs -+ * to based on the memory zone the pfn refers to */ -+static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, -+ u64 gpu_pfn) -+{ -+ struct rb_root *rbtree = NULL; -+ -+#ifdef CONFIG_64BIT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+#endif /* CONFIG_64BIT */ -+ if (gpu_pfn >= KBASE_REG_ZONE_CUSTOM_VA_BASE) -+ rbtree = &kctx->reg_rbtree_custom; -+ else if (gpu_pfn >= KBASE_REG_ZONE_EXEC_BASE) -+ rbtree = &kctx->reg_rbtree_exec; -+ else -+ rbtree = &kctx->reg_rbtree_same; -+#ifdef CONFIG_64BIT -+ } else { -+ if (gpu_pfn >= kctx->same_va_end) -+ rbtree = &kctx->reg_rbtree_custom; -+ else -+ rbtree = &kctx->reg_rbtree_same; -+ } -+#endif /* CONFIG_64BIT */ -+ -+ return rbtree; -+} -+ -+/* This function inserts a region into the tree. */ -+static void kbase_region_tracker_insert(struct kbase_context *kctx, -+ struct kbase_va_region *new_reg) -+{ -+ u64 start_pfn = new_reg->start_pfn; -+ struct rb_node **link = NULL; -+ struct rb_node *parent = NULL; -+ struct rb_root *rbtree = NULL; -+ -+ rbtree = kbase_reg_flags_to_rbtree(kctx, new_reg); -+ -+ link = &(rbtree->rb_node); -+ /* Find the right place in the tree using tree search */ -+ while (*link) { -+ struct kbase_va_region *old_reg; -+ -+ parent = *link; -+ old_reg = rb_entry(parent, struct kbase_va_region, rblink); -+ -+ /* RBTree requires no duplicate entries. */ -+ KBASE_DEBUG_ASSERT(old_reg->start_pfn != start_pfn); -+ -+ if (old_reg->start_pfn > start_pfn) -+ link = &(*link)->rb_left; -+ else -+ link = &(*link)->rb_right; -+ } -+ -+ /* Put the new node there, and rebalance tree */ -+ rb_link_node(&(new_reg->rblink), parent, link); -+ -+ rb_insert_color(&(new_reg->rblink), rbtree); -+} -+ -+/* Find allocated region enclosing free range. */ -+static struct kbase_va_region *kbase_region_tracker_find_region_enclosing_range_free( -+ struct kbase_context *kctx, u64 start_pfn, size_t nr_pages) -+{ -+ struct rb_node *rbnode = NULL; -+ struct kbase_va_region *reg = NULL; -+ struct rb_root *rbtree = NULL; -+ -+ u64 end_pfn = start_pfn + nr_pages; -+ -+ rbtree = kbase_gpu_va_to_rbtree(kctx, start_pfn); -+ -+ rbnode = rbtree->rb_node; -+ -+ while (rbnode) { -+ u64 tmp_start_pfn, tmp_end_pfn; -+ -+ reg = rb_entry(rbnode, struct kbase_va_region, rblink); -+ tmp_start_pfn = reg->start_pfn; -+ tmp_end_pfn = reg->start_pfn + reg->nr_pages; -+ -+ /* If start is lower than this, go left. */ -+ if (start_pfn < tmp_start_pfn) -+ rbnode = rbnode->rb_left; -+ /* If end is higher than this, then go right. */ -+ else if (end_pfn > tmp_end_pfn) -+ rbnode = rbnode->rb_right; -+ else /* Enclosing */ -+ return reg; -+ } -+ -+ return NULL; -+} -+ -+/* Find region enclosing given address. */ -+struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr) -+{ -+ struct rb_node *rbnode; -+ struct kbase_va_region *reg; -+ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; -+ struct rb_root *rbtree = NULL; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); -+ -+ rbnode = rbtree->rb_node; -+ -+ while (rbnode) { -+ u64 tmp_start_pfn, tmp_end_pfn; -+ -+ reg = rb_entry(rbnode, struct kbase_va_region, rblink); -+ tmp_start_pfn = reg->start_pfn; -+ tmp_end_pfn = reg->start_pfn + reg->nr_pages; -+ -+ /* If start is lower than this, go left. */ -+ if (gpu_pfn < tmp_start_pfn) -+ rbnode = rbnode->rb_left; -+ /* If end is higher than this, then go right. */ -+ else if (gpu_pfn >= tmp_end_pfn) -+ rbnode = rbnode->rb_right; -+ else /* Enclosing */ -+ return reg; -+ } -+ -+ return NULL; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_enclosing_address); -+ -+/* Find region with given base address */ -+struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr) -+{ -+ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; -+ struct rb_node *rbnode = NULL; -+ struct kbase_va_region *reg = NULL; -+ struct rb_root *rbtree = NULL; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ rbtree = kbase_gpu_va_to_rbtree(kctx, gpu_pfn); -+ -+ rbnode = rbtree->rb_node; -+ -+ while (rbnode) { -+ reg = rb_entry(rbnode, struct kbase_va_region, rblink); -+ if (reg->start_pfn > gpu_pfn) -+ rbnode = rbnode->rb_left; -+ else if (reg->start_pfn < gpu_pfn) -+ rbnode = rbnode->rb_right; -+ else -+ return reg; -+ -+ } -+ -+ return NULL; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_region_tracker_find_region_base_address); -+ -+/* Find region meeting given requirements */ -+static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs(struct kbase_context *kctx, struct kbase_va_region *reg_reqs, size_t nr_pages, size_t align) -+{ -+ struct rb_node *rbnode = NULL; -+ struct kbase_va_region *reg = NULL; -+ struct rb_root *rbtree = NULL; -+ -+ /* Note that this search is a linear search, as we do not have a target -+ address in mind, so does not benefit from the rbtree search */ -+ -+ rbtree = kbase_reg_flags_to_rbtree(kctx, reg_reqs); -+ -+ rbnode = rb_first(rbtree); -+ -+ while (rbnode) { -+ reg = rb_entry(rbnode, struct kbase_va_region, rblink); -+ if ((reg->nr_pages >= nr_pages) && -+ (reg->flags & KBASE_REG_FREE)) { -+ /* Check alignment */ -+ u64 start_pfn = (reg->start_pfn + align - 1) & ~(align - 1); -+ -+ if ((start_pfn >= reg->start_pfn) && -+ (start_pfn <= (reg->start_pfn + reg->nr_pages - 1)) && -+ ((start_pfn + nr_pages - 1) <= (reg->start_pfn + reg->nr_pages - 1))) -+ return reg; -+ } -+ rbnode = rb_next(rbnode); -+ } -+ -+ return NULL; -+} -+ -+/** -+ * @brief Remove a region object from the global list. -+ * -+ * The region reg is removed, possibly by merging with other free and -+ * compatible adjacent regions. It must be called with the context -+ * region lock held. The associated memory is not released (see -+ * kbase_free_alloced_region). Internal use only. -+ */ -+static int kbase_remove_va_region(struct kbase_context *kctx, struct kbase_va_region *reg) -+{ -+ struct rb_node *rbprev; -+ struct kbase_va_region *prev = NULL; -+ struct rb_node *rbnext; -+ struct kbase_va_region *next = NULL; -+ struct rb_root *reg_rbtree = NULL; -+ -+ int merged_front = 0; -+ int merged_back = 0; -+ int err = 0; -+ -+ reg_rbtree = kbase_reg_flags_to_rbtree(kctx, reg); -+ -+ /* Try to merge with the previous block first */ -+ rbprev = rb_prev(&(reg->rblink)); -+ if (rbprev) { -+ prev = rb_entry(rbprev, struct kbase_va_region, rblink); -+ if (prev->flags & KBASE_REG_FREE) { -+ /* We're compatible with the previous VMA, -+ * merge with it */ -+ WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != -+ (reg->flags & KBASE_REG_ZONE_MASK)); -+ prev->nr_pages += reg->nr_pages; -+ rb_erase(&(reg->rblink), reg_rbtree); -+ reg = prev; -+ merged_front = 1; -+ } -+ } -+ -+ /* Try to merge with the next block second */ -+ /* Note we do the lookup here as the tree may have been rebalanced. */ -+ rbnext = rb_next(&(reg->rblink)); -+ if (rbnext) { -+ /* We're compatible with the next VMA, merge with it */ -+ next = rb_entry(rbnext, struct kbase_va_region, rblink); -+ if (next->flags & KBASE_REG_FREE) { -+ WARN_ON((next->flags & KBASE_REG_ZONE_MASK) != -+ (reg->flags & KBASE_REG_ZONE_MASK)); -+ next->start_pfn = reg->start_pfn; -+ next->nr_pages += reg->nr_pages; -+ rb_erase(&(reg->rblink), reg_rbtree); -+ merged_back = 1; -+ if (merged_front) { -+ /* We already merged with prev, free it */ -+ kbase_free_alloced_region(reg); -+ } -+ } -+ } -+ -+ /* If we failed to merge then we need to add a new block */ -+ if (!(merged_front || merged_back)) { -+ /* -+ * We didn't merge anything. Add a new free -+ * placeholder and remove the original one. -+ */ -+ struct kbase_va_region *free_reg; -+ -+ free_reg = kbase_alloc_free_region(kctx, reg->start_pfn, reg->nr_pages, reg->flags & KBASE_REG_ZONE_MASK); -+ if (!free_reg) { -+ err = -ENOMEM; -+ goto out; -+ } -+ rb_replace_node(&(reg->rblink), &(free_reg->rblink), reg_rbtree); -+ } -+ -+ out: -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_remove_va_region); -+ -+/** -+ * @brief Insert a VA region to the list, replacing the current at_reg. -+ */ -+static int kbase_insert_va_region_nolock(struct kbase_context *kctx, struct kbase_va_region *new_reg, struct kbase_va_region *at_reg, u64 start_pfn, size_t nr_pages) -+{ -+ struct rb_root *reg_rbtree = NULL; -+ int err = 0; -+ -+ reg_rbtree = kbase_reg_flags_to_rbtree(kctx, at_reg); -+ -+ /* Must be a free region */ -+ KBASE_DEBUG_ASSERT((at_reg->flags & KBASE_REG_FREE) != 0); -+ /* start_pfn should be contained within at_reg */ -+ KBASE_DEBUG_ASSERT((start_pfn >= at_reg->start_pfn) && (start_pfn < at_reg->start_pfn + at_reg->nr_pages)); -+ /* at least nr_pages from start_pfn should be contained within at_reg */ -+ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= at_reg->start_pfn + at_reg->nr_pages); -+ -+ new_reg->start_pfn = start_pfn; -+ new_reg->nr_pages = nr_pages; -+ -+ /* Regions are a whole use, so swap and delete old one. */ -+ if (at_reg->start_pfn == start_pfn && at_reg->nr_pages == nr_pages) { -+ rb_replace_node(&(at_reg->rblink), &(new_reg->rblink), -+ reg_rbtree); -+ kbase_free_alloced_region(at_reg); -+ } -+ /* New region replaces the start of the old one, so insert before. */ -+ else if (at_reg->start_pfn == start_pfn) { -+ at_reg->start_pfn += nr_pages; -+ KBASE_DEBUG_ASSERT(at_reg->nr_pages >= nr_pages); -+ at_reg->nr_pages -= nr_pages; -+ -+ kbase_region_tracker_insert(kctx, new_reg); -+ } -+ /* New region replaces the end of the old one, so insert after. */ -+ else if ((at_reg->start_pfn + at_reg->nr_pages) == (start_pfn + nr_pages)) { -+ at_reg->nr_pages -= nr_pages; -+ -+ kbase_region_tracker_insert(kctx, new_reg); -+ } -+ /* New region splits the old one, so insert and create new */ -+ else { -+ struct kbase_va_region *new_front_reg; -+ -+ new_front_reg = kbase_alloc_free_region(kctx, -+ at_reg->start_pfn, -+ start_pfn - at_reg->start_pfn, -+ at_reg->flags & KBASE_REG_ZONE_MASK); -+ -+ if (new_front_reg) { -+ at_reg->nr_pages -= nr_pages + new_front_reg->nr_pages; -+ at_reg->start_pfn = start_pfn + nr_pages; -+ -+ kbase_region_tracker_insert(kctx, new_front_reg); -+ kbase_region_tracker_insert(kctx, new_reg); -+ } else { -+ err = -ENOMEM; -+ } -+ } -+ -+ return err; -+} -+ -+/** -+ * @brief Add a VA region to the list. -+ */ -+int kbase_add_va_region(struct kbase_context *kctx, -+ struct kbase_va_region *reg, u64 addr, -+ size_t nr_pages, size_t align) -+{ -+ struct kbase_va_region *tmp; -+ u64 gpu_pfn = addr >> PAGE_SHIFT; -+ int err = 0; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL != reg); -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ if (!align) -+ align = 1; -+ -+ /* must be a power of 2 */ -+ KBASE_DEBUG_ASSERT((align & (align - 1)) == 0); -+ KBASE_DEBUG_ASSERT(nr_pages > 0); -+ -+ /* Path 1: Map a specific address. Find the enclosing region, which *must* be free. */ -+ if (gpu_pfn) { -+ struct device *dev = kctx->kbdev->dev; -+ -+ KBASE_DEBUG_ASSERT(!(gpu_pfn & (align - 1))); -+ -+ tmp = kbase_region_tracker_find_region_enclosing_range_free(kctx, gpu_pfn, nr_pages); -+ if (!tmp) { -+ dev_warn(dev, "Enclosing region not found: 0x%08llx gpu_pfn, %zu nr_pages", gpu_pfn, nr_pages); -+ err = -ENOMEM; -+ goto exit; -+ } -+ if (!(tmp->flags & KBASE_REG_FREE)) { -+ dev_warn(dev, "Zone mismatch: %lu != %lu", tmp->flags & KBASE_REG_ZONE_MASK, reg->flags & KBASE_REG_ZONE_MASK); -+ dev_warn(dev, "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); -+ dev_warn(dev, "in function %s (%p, %p, 0x%llx, 0x%zx, 0x%zx)\n", __func__, kctx, reg, addr, nr_pages, align); -+ err = -ENOMEM; -+ goto exit; -+ } -+ -+ err = kbase_insert_va_region_nolock(kctx, reg, tmp, gpu_pfn, nr_pages); -+ if (err) { -+ dev_warn(dev, "Failed to insert va region"); -+ err = -ENOMEM; -+ goto exit; -+ } -+ -+ goto exit; -+ } -+ -+ /* Path 2: Map any free address which meets the requirements. */ -+ { -+ u64 start_pfn; -+ -+ /* -+ * Depending on the zone the allocation request is for -+ * we might need to retry it. -+ */ -+ do { -+ tmp = kbase_region_tracker_find_region_meeting_reqs( -+ kctx, reg, nr_pages, align); -+ if (tmp) { -+ start_pfn = (tmp->start_pfn + align - 1) & -+ ~(align - 1); -+ err = kbase_insert_va_region_nolock(kctx, reg, -+ tmp, start_pfn, nr_pages); -+ break; -+ } -+ -+ /* -+ * If the allocation is not from the same zone as JIT -+ * then don't retry, we're out of VA and there is -+ * nothing which can be done about it. -+ */ -+ if ((reg->flags & KBASE_REG_ZONE_MASK) != -+ KBASE_REG_ZONE_CUSTOM_VA) -+ break; -+ } while (kbase_jit_evict(kctx)); -+ -+ if (!tmp) -+ err = -ENOMEM; -+ } -+ -+ exit: -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_add_va_region); -+ -+/** -+ * @brief Initialize the internal region tracker data structure. -+ */ -+static void kbase_region_tracker_ds_init(struct kbase_context *kctx, -+ struct kbase_va_region *same_va_reg, -+ struct kbase_va_region *exec_reg, -+ struct kbase_va_region *custom_va_reg) -+{ -+ kctx->reg_rbtree_same = RB_ROOT; -+ kbase_region_tracker_insert(kctx, same_va_reg); -+ -+ /* Although exec and custom_va_reg don't always exist, -+ * initialize unconditionally because of the mem_view debugfs -+ * implementation which relies on these being empty */ -+ kctx->reg_rbtree_exec = RB_ROOT; -+ kctx->reg_rbtree_custom = RB_ROOT; -+ -+ if (exec_reg) -+ kbase_region_tracker_insert(kctx, exec_reg); -+ if (custom_va_reg) -+ kbase_region_tracker_insert(kctx, custom_va_reg); -+} -+ -+static void kbase_region_tracker_erase_rbtree(struct rb_root *rbtree) -+{ -+ struct rb_node *rbnode; -+ struct kbase_va_region *reg; -+ -+ do { -+ rbnode = rb_first(rbtree); -+ if (rbnode) { -+ rb_erase(rbnode, rbtree); -+ reg = rb_entry(rbnode, struct kbase_va_region, rblink); -+ kbase_free_alloced_region(reg); -+ } -+ } while (rbnode); -+} -+ -+void kbase_region_tracker_term(struct kbase_context *kctx) -+{ -+ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); -+ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); -+ kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); -+} -+ -+/** -+ * Initialize the region tracker data structure. -+ */ -+int kbase_region_tracker_init(struct kbase_context *kctx) -+{ -+ struct kbase_va_region *same_va_reg; -+ struct kbase_va_region *exec_reg = NULL; -+ struct kbase_va_region *custom_va_reg = NULL; -+ size_t same_va_bits = sizeof(void *) * BITS_PER_BYTE; -+ u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; -+ u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; -+ u64 same_va_pages; -+ int err; -+ -+ /* Take the lock as kbase_free_alloced_region requires it */ -+ kbase_gpu_vm_lock(kctx); -+ -+#if defined(CONFIG_ARM64) -+ same_va_bits = VA_BITS; -+#elif defined(CONFIG_X86_64) -+ same_va_bits = 47; -+#elif defined(CONFIG_64BIT) -+#error Unsupported 64-bit architecture -+#endif -+ -+#ifdef CONFIG_64BIT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ same_va_bits = 32; -+ else if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) -+ same_va_bits = 33; -+#endif -+ -+ if (kctx->kbdev->gpu_props.mmu.va_bits < same_va_bits) { -+ err = -EINVAL; -+ goto fail_unlock; -+ } -+ -+ same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; -+ /* all have SAME_VA */ -+ same_va_reg = kbase_alloc_free_region(kctx, 1, -+ same_va_pages, -+ KBASE_REG_ZONE_SAME_VA); -+ -+ if (!same_va_reg) { -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+#ifdef CONFIG_64BIT -+ /* 32-bit clients have exec and custom VA zones */ -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+#endif -+ if (gpu_va_limit <= KBASE_REG_ZONE_CUSTOM_VA_BASE) { -+ err = -EINVAL; -+ goto fail_free_same_va; -+ } -+ /* If the current size of TMEM is out of range of the -+ * virtual address space addressable by the MMU then -+ * we should shrink it to fit -+ */ -+ if ((KBASE_REG_ZONE_CUSTOM_VA_BASE + KBASE_REG_ZONE_CUSTOM_VA_SIZE) >= gpu_va_limit) -+ custom_va_size = gpu_va_limit - KBASE_REG_ZONE_CUSTOM_VA_BASE; -+ -+ exec_reg = kbase_alloc_free_region(kctx, -+ KBASE_REG_ZONE_EXEC_BASE, -+ KBASE_REG_ZONE_EXEC_SIZE, -+ KBASE_REG_ZONE_EXEC); -+ -+ if (!exec_reg) { -+ err = -ENOMEM; -+ goto fail_free_same_va; -+ } -+ -+ custom_va_reg = kbase_alloc_free_region(kctx, -+ KBASE_REG_ZONE_CUSTOM_VA_BASE, -+ custom_va_size, KBASE_REG_ZONE_CUSTOM_VA); -+ -+ if (!custom_va_reg) { -+ err = -ENOMEM; -+ goto fail_free_exec; -+ } -+#ifdef CONFIG_64BIT -+ } -+#endif -+ -+ kbase_region_tracker_ds_init(kctx, same_va_reg, exec_reg, custom_va_reg); -+ -+ kctx->same_va_end = same_va_pages + 1; -+ -+ kbase_gpu_vm_unlock(kctx); -+ return 0; -+ -+fail_free_exec: -+ kbase_free_alloced_region(exec_reg); -+fail_free_same_va: -+ kbase_free_alloced_region(same_va_reg); -+fail_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return err; -+} -+ -+int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages) -+{ -+#ifdef CONFIG_64BIT -+ struct kbase_va_region *same_va; -+ struct kbase_va_region *custom_va_reg; -+ u64 same_va_bits; -+ u64 total_va_size; -+ int err; -+ -+ /* -+ * Nothing to do for 32-bit clients, JIT uses the existing -+ * custom VA zone. -+ */ -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ return 0; -+ -+#if defined(CONFIG_ARM64) -+ same_va_bits = VA_BITS; -+#elif defined(CONFIG_X86_64) -+ same_va_bits = 47; -+#elif defined(CONFIG_64BIT) -+#error Unsupported 64-bit architecture -+#endif -+ -+ if (kbase_hw_has_feature(kctx->kbdev, BASE_HW_FEATURE_33BIT_VA)) -+ same_va_bits = 33; -+ -+ total_va_size = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ /* -+ * Modify the same VA free region after creation. Be careful to ensure -+ * that allocations haven't been made as they could cause an overlap -+ * to happen with existing same VA allocations and the custom VA zone. -+ */ -+ same_va = kbase_region_tracker_find_region_base_address(kctx, -+ PAGE_SIZE); -+ if (!same_va) { -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ /* The region flag or region size has changed since creation so bail. */ -+ if ((!(same_va->flags & KBASE_REG_FREE)) || -+ (same_va->nr_pages != total_va_size)) { -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ if (same_va->nr_pages < jit_va_pages || -+ kctx->same_va_end < jit_va_pages) { -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ /* It's safe to adjust the same VA zone now */ -+ same_va->nr_pages -= jit_va_pages; -+ kctx->same_va_end -= jit_va_pages; -+ -+ /* -+ * Create a custom VA zone at the end of the VA for allocations which -+ * JIT can use so it doesn't have to allocate VA from the kernel. -+ */ -+ custom_va_reg = kbase_alloc_free_region(kctx, -+ kctx->same_va_end, -+ jit_va_pages, -+ KBASE_REG_ZONE_CUSTOM_VA); -+ -+ if (!custom_va_reg) { -+ /* -+ * The context will be destroyed if we fail here so no point -+ * reverting the change we made to same_va. -+ */ -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ kbase_region_tracker_insert(kctx, custom_va_reg); -+ -+ kbase_gpu_vm_unlock(kctx); -+ return 0; -+ -+fail_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return err; -+#else -+ return 0; -+#endif -+} -+ -+int kbase_mem_init(struct kbase_device *kbdev) -+{ -+ struct kbasep_mem_device *memdev; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ memdev = &kbdev->memdev; -+ kbdev->mem_pool_max_size_default = KBASE_MEM_POOL_MAX_SIZE_KCTX; -+ -+ /* Initialize memory usage */ -+ atomic_set(&memdev->used_pages, 0); -+ -+ return kbase_mem_pool_init(&kbdev->mem_pool, -+ KBASE_MEM_POOL_MAX_SIZE_KBDEV, kbdev, NULL); -+} -+ -+void kbase_mem_halt(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+void kbase_mem_term(struct kbase_device *kbdev) -+{ -+ struct kbasep_mem_device *memdev; -+ int pages; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ memdev = &kbdev->memdev; -+ -+ pages = atomic_read(&memdev->used_pages); -+ if (pages != 0) -+ dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); -+ -+ kbase_mem_pool_term(&kbdev->mem_pool); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mem_term); -+ -+ -+ -+ -+/** -+ * @brief Allocate a free region object. -+ * -+ * The allocated object is not part of any list yet, and is flagged as -+ * KBASE_REG_FREE. No mapping is allocated yet. -+ * -+ * zone is KBASE_REG_ZONE_CUSTOM_VA, KBASE_REG_ZONE_SAME_VA, or KBASE_REG_ZONE_EXEC -+ * -+ */ -+struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone) -+{ -+ struct kbase_va_region *new_reg; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ /* zone argument should only contain zone related region flags */ -+ KBASE_DEBUG_ASSERT((zone & ~KBASE_REG_ZONE_MASK) == 0); -+ KBASE_DEBUG_ASSERT(nr_pages > 0); -+ /* 64-bit address range is the max */ -+ KBASE_DEBUG_ASSERT(start_pfn + nr_pages <= (U64_MAX / PAGE_SIZE)); -+ -+ new_reg = kzalloc(sizeof(*new_reg), GFP_KERNEL); -+ -+ if (!new_reg) -+ return NULL; -+ -+ new_reg->cpu_alloc = NULL; /* no alloc bound yet */ -+ new_reg->gpu_alloc = NULL; /* no alloc bound yet */ -+ new_reg->kctx = kctx; -+ new_reg->flags = zone | KBASE_REG_FREE; -+ -+ new_reg->flags |= KBASE_REG_GROWABLE; -+ -+ new_reg->start_pfn = start_pfn; -+ new_reg->nr_pages = nr_pages; -+ -+ return new_reg; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_alloc_free_region); -+ -+/** -+ * @brief Free a region object. -+ * -+ * The described region must be freed of any mapping. -+ * -+ * If the region is not flagged as KBASE_REG_FREE, the region's -+ * alloc object will be released. -+ * It is a bug if no alloc object exists for non-free regions. -+ * -+ */ -+void kbase_free_alloced_region(struct kbase_va_region *reg) -+{ -+ if (!(reg->flags & KBASE_REG_FREE)) { -+ /* -+ * The physical allocation should have been removed from the -+ * eviction list before this function is called. However, in the -+ * case of abnormal process termination or the app leaking the -+ * memory kbase_mem_free_region is not called so it can still be -+ * on the list at termination time of the region tracker. -+ */ -+ if (!list_empty(®->gpu_alloc->evict_node)) { -+ /* -+ * Unlink the physical allocation before unmaking it -+ * evictable so that the allocation isn't grown back to -+ * its last backed size as we're going to unmap it -+ * anyway. -+ */ -+ reg->cpu_alloc->reg = NULL; -+ if (reg->cpu_alloc != reg->gpu_alloc) -+ reg->gpu_alloc->reg = NULL; -+ -+ /* -+ * If a region has been made evictable then we must -+ * unmake it before trying to free it. -+ * If the memory hasn't been reclaimed it will be -+ * unmapped and freed below, if it has been reclaimed -+ * then the operations below are no-ops. -+ */ -+ if (reg->flags & KBASE_REG_DONT_NEED) { -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == -+ KBASE_MEM_TYPE_NATIVE); -+ kbase_mem_evictable_unmake(reg->gpu_alloc); -+ } -+ } -+ -+ /* -+ * Remove the region from the sticky resource metadata -+ * list should it be there. -+ */ -+ kbase_sticky_resource_release(reg->kctx, NULL, -+ reg->start_pfn << PAGE_SHIFT); -+ -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+ /* To detect use-after-free in debug builds */ -+ KBASE_DEBUG_CODE(reg->flags |= KBASE_REG_FREE); -+ } -+ kfree(reg); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_free_alloced_region); -+ -+int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align) -+{ -+ int err; -+ size_t i = 0; -+ unsigned long attr; -+ unsigned long mask = ~KBASE_REG_MEMATTR_MASK; -+ -+ if ((kctx->kbdev->system_coherency == COHERENCY_ACE) && -+ (reg->flags & KBASE_REG_SHARE_BOTH)) -+ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_OUTER_WA); -+ else -+ attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL != reg); -+ -+ err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); -+ if (err) -+ return err; -+ -+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { -+ u64 stride; -+ struct kbase_mem_phy_alloc *alloc; -+ -+ alloc = reg->gpu_alloc; -+ stride = alloc->imported.alias.stride; -+ KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); -+ for (i = 0; i < alloc->imported.alias.nents; i++) { -+ if (alloc->imported.alias.aliased[i].alloc) { -+ err = kbase_mmu_insert_pages(kctx, -+ reg->start_pfn + (i * stride), -+ alloc->imported.alias.aliased[i].alloc->pages + alloc->imported.alias.aliased[i].offset, -+ alloc->imported.alias.aliased[i].length, -+ reg->flags); -+ if (err) -+ goto bad_insert; -+ -+ kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); -+ } else { -+ err = kbase_mmu_insert_single_page(kctx, -+ reg->start_pfn + i * stride, -+ page_to_phys(kctx->aliasing_sink_page), -+ alloc->imported.alias.aliased[i].length, -+ (reg->flags & mask) | attr); -+ -+ if (err) -+ goto bad_insert; -+ } -+ } -+ } else { -+ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, -+ kbase_get_gpu_phy_pages(reg), -+ kbase_reg_current_backed_size(reg), -+ reg->flags); -+ if (err) -+ goto bad_insert; -+ kbase_mem_phy_alloc_gpu_mapped(reg->gpu_alloc); -+ } -+ -+ return err; -+ -+bad_insert: -+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { -+ u64 stride; -+ -+ stride = reg->gpu_alloc->imported.alias.stride; -+ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); -+ while (i--) -+ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) { -+ kbase_mmu_teardown_pages(kctx, reg->start_pfn + (i * stride), reg->gpu_alloc->imported.alias.aliased[i].length); -+ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); -+ } -+ } -+ -+ kbase_remove_va_region(kctx, reg); -+ -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_gpu_mmap); -+ -+static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc, bool writeable); -+ -+int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) -+{ -+ int err; -+ -+ if (reg->start_pfn == 0) -+ return 0; -+ -+ if (reg->gpu_alloc && reg->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { -+ size_t i; -+ -+ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, reg->nr_pages); -+ KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); -+ for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) -+ if (reg->gpu_alloc->imported.alias.aliased[i].alloc) -+ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); -+ } else { -+ err = kbase_mmu_teardown_pages(kctx, reg->start_pfn, kbase_reg_current_backed_size(reg)); -+ kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc); -+ } -+ -+ if (reg->gpu_alloc && reg->gpu_alloc->type == -+ KBASE_MEM_TYPE_IMPORTED_USER_BUF) { -+ struct kbase_alloc_import_user_buf *user_buf = -+ ®->gpu_alloc->imported.user_buf; -+ -+ if (user_buf->current_mapping_usage_count & PINNED_ON_IMPORT) { -+ user_buf->current_mapping_usage_count &= -+ ~PINNED_ON_IMPORT; -+ -+ kbase_jd_user_buf_unmap(kctx, reg->gpu_alloc, -+ (reg->flags & KBASE_REG_GPU_WR)); -+ } -+ } -+ -+ if (err) -+ return err; -+ -+ err = kbase_remove_va_region(kctx, reg); -+ return err; -+} -+ -+static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( -+ struct kbase_context *kctx, -+ unsigned long uaddr, size_t size, u64 *offset) -+{ -+ struct vm_area_struct *vma; -+ struct kbase_cpu_mapping *map; -+ unsigned long vm_pgoff_in_region; -+ unsigned long vm_off_in_region; -+ unsigned long map_start; -+ size_t map_size; -+ -+ lockdep_assert_held(¤t->mm->mmap_lock); -+ -+ if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ -+ return NULL; -+ -+ vma = find_vma_intersection(current->mm, uaddr, uaddr+size); -+ -+ if (!vma || vma->vm_start > uaddr) -+ return NULL; -+ if (vma->vm_ops != &kbase_vm_ops) -+ /* Not ours! */ -+ return NULL; -+ -+ map = vma->vm_private_data; -+ -+ if (map->kctx != kctx) -+ /* Not from this context! */ -+ return NULL; -+ -+ vm_pgoff_in_region = vma->vm_pgoff - map->region->start_pfn; -+ vm_off_in_region = vm_pgoff_in_region << PAGE_SHIFT; -+ map_start = vma->vm_start - vm_off_in_region; -+ map_size = map->region->nr_pages << PAGE_SHIFT; -+ -+ if ((uaddr + size) > (map_start + map_size)) -+ /* Not within the CPU mapping */ -+ return NULL; -+ -+ *offset = (uaddr - vma->vm_start) + vm_off_in_region; -+ -+ return map; -+} -+ -+int kbasep_find_enclosing_cpu_mapping_offset( -+ struct kbase_context *kctx, -+ unsigned long uaddr, size_t size, u64 *offset) -+{ -+ struct kbase_cpu_mapping *map; -+ -+ kbase_os_mem_map_lock(kctx); -+ -+ map = kbasep_find_enclosing_cpu_mapping(kctx, uaddr, size, offset); -+ -+ kbase_os_mem_map_unlock(kctx); -+ -+ if (!map) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbasep_find_enclosing_cpu_mapping_offset); -+ -+void kbase_sync_single(struct kbase_context *kctx, -+ phys_addr_t cpu_pa, phys_addr_t gpu_pa, -+ off_t offset, size_t size, enum kbase_sync_type sync_fn) -+{ -+ struct page *cpu_page; -+ -+ cpu_page = pfn_to_page(PFN_DOWN(cpu_pa)); -+ -+ if (likely(cpu_pa == gpu_pa)) { -+ dma_addr_t dma_addr; -+ -+ BUG_ON(!cpu_page); -+ BUG_ON(offset + size > PAGE_SIZE); -+ -+ dma_addr = kbase_dma_addr(cpu_page) + offset; -+ if (sync_fn == KBASE_SYNC_TO_CPU) -+ dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, -+ size, DMA_BIDIRECTIONAL); -+ else if (sync_fn == KBASE_SYNC_TO_DEVICE) -+ dma_sync_single_for_device(kctx->kbdev->dev, dma_addr, -+ size, DMA_BIDIRECTIONAL); -+ } else { -+ void *src = NULL; -+ void *dst = NULL; -+ struct page *gpu_page; -+ -+ if (WARN(!gpu_pa, "No GPU PA found for infinite cache op")) -+ return; -+ -+ gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); -+ -+ if (sync_fn == KBASE_SYNC_TO_DEVICE) { -+ src = ((unsigned char *)kmap(cpu_page)) + offset; -+ dst = ((unsigned char *)kmap(gpu_page)) + offset; -+ } else if (sync_fn == KBASE_SYNC_TO_CPU) { -+ dma_sync_single_for_cpu(kctx->kbdev->dev, -+ kbase_dma_addr(gpu_page) + offset, -+ size, DMA_BIDIRECTIONAL); -+ src = ((unsigned char *)kmap(gpu_page)) + offset; -+ dst = ((unsigned char *)kmap(cpu_page)) + offset; -+ } -+ memcpy(dst, src, size); -+ kunmap(gpu_page); -+ kunmap(cpu_page); -+ if (sync_fn == KBASE_SYNC_TO_DEVICE) -+ dma_sync_single_for_device(kctx->kbdev->dev, -+ kbase_dma_addr(gpu_page) + offset, -+ size, DMA_BIDIRECTIONAL); -+ } -+} -+ -+static int kbase_do_syncset(struct kbase_context *kctx, -+ struct basep_syncset *sset, enum kbase_sync_type sync_fn) -+{ -+ int err = 0; -+ struct kbase_va_region *reg; -+ struct kbase_cpu_mapping *map; -+ unsigned long start; -+ size_t size; -+ phys_addr_t *cpu_pa; -+ phys_addr_t *gpu_pa; -+ u64 page_off, page_count; -+ u64 i; -+ u64 offset; -+ -+ kbase_os_mem_map_lock(kctx); -+ kbase_gpu_vm_lock(kctx); -+ -+ /* find the region where the virtual address is contained */ -+ reg = kbase_region_tracker_find_region_enclosing_address(kctx, -+ sset->mem_handle.basep.handle); -+ if (!reg) { -+ dev_warn(kctx->kbdev->dev, "Can't find region at VA 0x%016llX", -+ sset->mem_handle.basep.handle); -+ err = -EINVAL; -+ goto out_unlock; -+ } -+ -+ if (!(reg->flags & KBASE_REG_CPU_CACHED)) -+ goto out_unlock; -+ -+ start = (uintptr_t)sset->user_addr; -+ size = (size_t)sset->size; -+ -+ map = kbasep_find_enclosing_cpu_mapping(kctx, start, size, &offset); -+ if (!map) { -+ dev_warn(kctx->kbdev->dev, "Can't find CPU mapping 0x%016lX for VA 0x%016llX", -+ start, sset->mem_handle.basep.handle); -+ err = -EINVAL; -+ goto out_unlock; -+ } -+ -+ page_off = offset >> PAGE_SHIFT; -+ offset &= ~PAGE_MASK; -+ page_count = (size + offset + (PAGE_SIZE - 1)) >> PAGE_SHIFT; -+ cpu_pa = kbase_get_cpu_phy_pages(reg); -+ gpu_pa = kbase_get_gpu_phy_pages(reg); -+ -+ if (page_off > reg->nr_pages || -+ page_off + page_count > reg->nr_pages) { -+ /* Sync overflows the region */ -+ err = -EINVAL; -+ goto out_unlock; -+ } -+ -+ /* Sync first page */ -+ if (cpu_pa[page_off]) { -+ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); -+ -+ kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], -+ offset, sz, sync_fn); -+ } -+ -+ /* Sync middle pages (if any) */ -+ for (i = 1; page_count > 2 && i < page_count - 1; i++) { -+ /* we grow upwards, so bail on first non-present page */ -+ if (!cpu_pa[page_off + i]) -+ break; -+ -+ kbase_sync_single(kctx, cpu_pa[page_off + i], -+ gpu_pa[page_off + i], 0, PAGE_SIZE, sync_fn); -+ } -+ -+ /* Sync last page (if any) */ -+ if (page_count > 1 && cpu_pa[page_off + page_count - 1]) { -+ size_t sz = ((start + size - 1) & ~PAGE_MASK) + 1; -+ -+ kbase_sync_single(kctx, cpu_pa[page_off + page_count - 1], -+ gpu_pa[page_off + page_count - 1], 0, sz, -+ sync_fn); -+ } -+ -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ kbase_os_mem_map_unlock(kctx); -+ return err; -+} -+ -+int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) -+{ -+ int err = -EINVAL; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(sset != NULL); -+ -+ if (sset->mem_handle.basep.handle & ~PAGE_MASK) { -+ dev_warn(kctx->kbdev->dev, -+ "mem_handle: passed parameter is invalid"); -+ return -EINVAL; -+ } -+ -+ switch (sset->type) { -+ case BASE_SYNCSET_OP_MSYNC: -+ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_DEVICE); -+ break; -+ -+ case BASE_SYNCSET_OP_CSYNC: -+ err = kbase_do_syncset(kctx, sset, KBASE_SYNC_TO_CPU); -+ break; -+ -+ default: -+ dev_warn(kctx->kbdev->dev, "Unknown msync op %d\n", sset->type); -+ break; -+ } -+ -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_sync_now); -+ -+/* vm lock must be held */ -+int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg) -+{ -+ int err; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL != reg); -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* -+ * Unlink the physical allocation before unmaking it evictable so -+ * that the allocation isn't grown back to its last backed size -+ * as we're going to unmap it anyway. -+ */ -+ reg->cpu_alloc->reg = NULL; -+ if (reg->cpu_alloc != reg->gpu_alloc) -+ reg->gpu_alloc->reg = NULL; -+ -+ /* -+ * If a region has been made evictable then we must unmake it -+ * before trying to free it. -+ * If the memory hasn't been reclaimed it will be unmapped and freed -+ * below, if it has been reclaimed then the operations below are no-ops. -+ */ -+ if (reg->flags & KBASE_REG_DONT_NEED) { -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->type == -+ KBASE_MEM_TYPE_NATIVE); -+ kbase_mem_evictable_unmake(reg->gpu_alloc); -+ } -+ -+ err = kbase_gpu_munmap(kctx, reg); -+ if (err) { -+ dev_warn(reg->kctx->kbdev->dev, "Could not unmap from the GPU...\n"); -+ goto out; -+ } -+ -+ /* This will also free the physical pages */ -+ kbase_free_alloced_region(reg); -+ -+ out: -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mem_free_region); -+ -+/** -+ * @brief Free the region from the GPU and unregister it. -+ * -+ * This function implements the free operation on a memory segment. -+ * It will loudly fail if called with outstanding mappings. -+ */ -+int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) -+{ -+ int err = 0; -+ struct kbase_va_region *reg; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ -+ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { -+ dev_warn(kctx->kbdev->dev, "kbase_mem_free: gpu_addr parameter is invalid"); -+ return -EINVAL; -+ } -+ -+ if (0 == gpu_addr) { -+ dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); -+ return -EINVAL; -+ } -+ kbase_gpu_vm_lock(kctx); -+ -+ if (gpu_addr >= BASE_MEM_COOKIE_BASE && -+ gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { -+ int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); -+ -+ reg = kctx->pending_regions[cookie]; -+ if (!reg) { -+ err = -EINVAL; -+ goto out_unlock; -+ } -+ -+ /* ask to unlink the cookie as we'll free it */ -+ -+ kctx->pending_regions[cookie] = NULL; -+ kctx->cookies |= (1UL << cookie); -+ -+ kbase_free_alloced_region(reg); -+ } else { -+ /* A real GPU va */ -+ /* Validate the region */ -+ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); -+ if (!reg || (reg->flags & KBASE_REG_FREE)) { -+ dev_warn(kctx->kbdev->dev, "kbase_mem_free called with nonexistent gpu_addr 0x%llX", -+ gpu_addr); -+ err = -EINVAL; -+ goto out_unlock; -+ } -+ -+ if ((reg->flags & KBASE_REG_ZONE_MASK) == KBASE_REG_ZONE_SAME_VA) { -+ /* SAME_VA must be freed through munmap */ -+ dev_warn(kctx->kbdev->dev, "%s called on SAME_VA memory 0x%llX", __func__, -+ gpu_addr); -+ err = -EINVAL; -+ goto out_unlock; -+ } -+ err = kbase_mem_free_region(kctx, reg); -+ } -+ -+ out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mem_free); -+ -+int kbase_update_region_flags(struct kbase_context *kctx, -+ struct kbase_va_region *reg, unsigned long flags) -+{ -+ KBASE_DEBUG_ASSERT(NULL != reg); -+ KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); -+ -+ reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); -+ /* all memory is now growable */ -+ reg->flags |= KBASE_REG_GROWABLE; -+ -+ if (flags & BASE_MEM_GROW_ON_GPF) -+ reg->flags |= KBASE_REG_PF_GROW; -+ -+ if (flags & BASE_MEM_PROT_CPU_WR) -+ reg->flags |= KBASE_REG_CPU_WR; -+ -+ if (flags & BASE_MEM_PROT_CPU_RD) -+ reg->flags |= KBASE_REG_CPU_RD; -+ -+ if (flags & BASE_MEM_PROT_GPU_WR) -+ reg->flags |= KBASE_REG_GPU_WR; -+ -+ if (flags & BASE_MEM_PROT_GPU_RD) -+ reg->flags |= KBASE_REG_GPU_RD; -+ -+ if (0 == (flags & BASE_MEM_PROT_GPU_EX)) -+ reg->flags |= KBASE_REG_GPU_NX; -+ -+ if (!kbase_device_is_cpu_coherent(kctx->kbdev)) { -+ if (flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) -+ return -EINVAL; -+ } else if (flags & (BASE_MEM_COHERENT_SYSTEM | -+ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { -+ reg->flags |= KBASE_REG_SHARE_BOTH; -+ } -+ -+ if (!(reg->flags & KBASE_REG_SHARE_BOTH) && -+ flags & BASE_MEM_COHERENT_LOCAL) { -+ reg->flags |= KBASE_REG_SHARE_IN; -+ } -+ -+ /* Set up default MEMATTR usage */ -+ if (kctx->kbdev->system_coherency == COHERENCY_ACE && -+ (reg->flags & KBASE_REG_SHARE_BOTH)) { -+ reg->flags |= -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); -+ } else { -+ reg->flags |= -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT); -+ } -+ -+ return 0; -+} -+ -+int kbase_alloc_phy_pages_helper( -+ struct kbase_mem_phy_alloc *alloc, -+ size_t nr_pages_requested) -+{ -+ int new_page_count __maybe_unused; -+ size_t old_page_count = alloc->nents; -+ -+ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); -+ KBASE_DEBUG_ASSERT(alloc->imported.kctx); -+ -+ if (nr_pages_requested == 0) -+ goto done; /*nothing to do*/ -+ -+ new_page_count = kbase_atomic_add_pages( -+ nr_pages_requested, &alloc->imported.kctx->used_pages); -+ kbase_atomic_add_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages); -+ -+ /* Increase mm counters before we allocate pages so that this -+ * allocation is visible to the OOM killer */ -+ kbase_process_page_usage_inc(alloc->imported.kctx, nr_pages_requested); -+ -+ if (kbase_mem_pool_alloc_pages(&alloc->imported.kctx->mem_pool, -+ nr_pages_requested, alloc->pages + old_page_count) != 0) -+ goto no_alloc; -+ -+ KBASE_TLSTREAM_AUX_PAGESALLOC( -+ (u32)alloc->imported.kctx->id, -+ (u64)new_page_count); -+ -+ alloc->nents += nr_pages_requested; -+done: -+ return 0; -+ -+no_alloc: -+ kbase_process_page_usage_dec(alloc->imported.kctx, nr_pages_requested); -+ kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->used_pages); -+ kbase_atomic_sub_pages(nr_pages_requested, &alloc->imported.kctx->kbdev->memdev.used_pages); -+ -+ return -ENOMEM; -+} -+ -+int kbase_free_phy_pages_helper( -+ struct kbase_mem_phy_alloc *alloc, -+ size_t nr_pages_to_free) -+{ -+ struct kbase_context *kctx = alloc->imported.kctx; -+ bool syncback; -+ bool reclaimed = (alloc->evicted != 0); -+ phys_addr_t *start_free; -+ int new_page_count __maybe_unused; -+ -+ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_NATIVE); -+ KBASE_DEBUG_ASSERT(alloc->imported.kctx); -+ KBASE_DEBUG_ASSERT(alloc->nents >= nr_pages_to_free); -+ -+ /* early out if nothing to do */ -+ if (0 == nr_pages_to_free) -+ return 0; -+ -+ start_free = alloc->pages + alloc->nents - nr_pages_to_free; -+ -+ syncback = alloc->properties & KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; -+ -+ kbase_mem_pool_free_pages(&kctx->mem_pool, -+ nr_pages_to_free, -+ start_free, -+ syncback, -+ reclaimed); -+ -+ alloc->nents -= nr_pages_to_free; -+ -+ /* -+ * If the allocation was not evicted (i.e. evicted == 0) then -+ * the page accounting needs to be done. -+ */ -+ if (!reclaimed) { -+ kbase_process_page_usage_dec(kctx, nr_pages_to_free); -+ new_page_count = kbase_atomic_sub_pages(nr_pages_to_free, -+ &kctx->used_pages); -+ kbase_atomic_sub_pages(nr_pages_to_free, -+ &kctx->kbdev->memdev.used_pages); -+ -+ KBASE_TLSTREAM_AUX_PAGESALLOC( -+ (u32)kctx->id, -+ (u64)new_page_count); -+ } -+ -+ return 0; -+} -+ -+void kbase_mem_kref_free(struct kref *kref) -+{ -+ struct kbase_mem_phy_alloc *alloc; -+ -+ alloc = container_of(kref, struct kbase_mem_phy_alloc, kref); -+ -+ switch (alloc->type) { -+ case KBASE_MEM_TYPE_NATIVE: { -+ WARN_ON(!alloc->imported.kctx); -+ /* -+ * The physical allocation must have been removed from the -+ * eviction list before trying to free it. -+ */ -+ WARN_ON(!list_empty(&alloc->evict_node)); -+ kbase_free_phy_pages_helper(alloc, alloc->nents); -+ break; -+ } -+ case KBASE_MEM_TYPE_ALIAS: { -+ /* just call put on the underlying phy allocs */ -+ size_t i; -+ struct kbase_aliased *aliased; -+ -+ aliased = alloc->imported.alias.aliased; -+ if (aliased) { -+ for (i = 0; i < alloc->imported.alias.nents; i++) -+ if (aliased[i].alloc) -+ kbase_mem_phy_alloc_put(aliased[i].alloc); -+ vfree(aliased); -+ } -+ break; -+ } -+ case KBASE_MEM_TYPE_RAW: -+ /* raw pages, external cleanup */ -+ break; -+ #ifdef CONFIG_UMP -+ case KBASE_MEM_TYPE_IMPORTED_UMP: -+ ump_dd_release(alloc->imported.ump_handle); -+ break; -+#endif -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ case KBASE_MEM_TYPE_IMPORTED_UMM: -+ dma_buf_detach(alloc->imported.umm.dma_buf, -+ alloc->imported.umm.dma_attachment); -+ dma_buf_put(alloc->imported.umm.dma_buf); -+ break; -+#endif -+ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: -+ if (alloc->imported.user_buf.mm) -+ mmdrop(alloc->imported.user_buf.mm); -+ kfree(alloc->imported.user_buf.pages); -+ break; -+ case KBASE_MEM_TYPE_TB:{ -+ void *tb; -+ -+ tb = alloc->imported.kctx->jctx.tb; -+ kbase_device_trace_buffer_uninstall(alloc->imported.kctx); -+ vfree(tb); -+ break; -+ } -+ default: -+ WARN(1, "Unexecpted free of type %d\n", alloc->type); -+ break; -+ } -+ -+ /* Free based on allocation type */ -+ if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) -+ vfree(alloc); -+ else -+ kfree(alloc); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mem_kref_free); -+ -+int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) -+{ -+ KBASE_DEBUG_ASSERT(NULL != reg); -+ KBASE_DEBUG_ASSERT(vsize > 0); -+ -+ /* validate user provided arguments */ -+ if (size > vsize || vsize > reg->nr_pages) -+ goto out_term; -+ -+ /* Prevent vsize*sizeof from wrapping around. -+ * For instance, if vsize is 2**29+1, we'll allocate 1 byte and the alloc won't fail. -+ */ -+ if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) -+ goto out_term; -+ -+ KBASE_DEBUG_ASSERT(0 != vsize); -+ -+ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) -+ goto out_term; -+ -+ reg->cpu_alloc->reg = reg; -+ if (reg->cpu_alloc != reg->gpu_alloc) { -+ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, size) != 0) -+ goto out_rollback; -+ reg->gpu_alloc->reg = reg; -+ } -+ -+ return 0; -+ -+out_rollback: -+ kbase_free_phy_pages_helper(reg->cpu_alloc, size); -+out_term: -+ return -1; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); -+ -+bool kbase_check_alloc_flags(unsigned long flags) -+{ -+ /* Only known input flags should be set. */ -+ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) -+ return false; -+ -+ /* At least one flag should be set */ -+ if (flags == 0) -+ return false; -+ -+ /* Either the GPU or CPU must be reading from the allocated memory */ -+ if ((flags & (BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD)) == 0) -+ return false; -+ -+ /* Either the GPU or CPU must be writing to the allocated memory */ -+ if ((flags & (BASE_MEM_PROT_CPU_WR | BASE_MEM_PROT_GPU_WR)) == 0) -+ return false; -+ -+ /* GPU cannot be writing to GPU executable memory and cannot grow the memory on page fault. */ -+ if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) -+ return false; -+ -+ /* GPU should have at least read or write access otherwise there is no -+ reason for allocating. */ -+ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) -+ return false; -+ -+ /* BASE_MEM_IMPORT_SHARED is only valid for imported memory */ -+ if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) -+ return false; -+ -+ return true; -+} -+ -+bool kbase_check_import_flags(unsigned long flags) -+{ -+ /* Only known input flags should be set. */ -+ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) -+ return false; -+ -+ /* At least one flag should be set */ -+ if (flags == 0) -+ return false; -+ -+ /* Imported memory cannot be GPU executable */ -+ if (flags & BASE_MEM_PROT_GPU_EX) -+ return false; -+ -+ /* Imported memory cannot grow on page fault */ -+ if (flags & BASE_MEM_GROW_ON_GPF) -+ return false; -+ -+ /* GPU should have at least read or write access otherwise there is no -+ reason for importing. */ -+ if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) -+ return false; -+ -+ /* Secure memory cannot be read by the CPU */ -+ if ((flags & BASE_MEM_SECURE) && (flags & BASE_MEM_PROT_CPU_RD)) -+ return false; -+ -+ return true; -+} -+ -+/** -+ * @brief Acquire the per-context region list lock -+ */ -+void kbase_gpu_vm_lock(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ mutex_lock(&kctx->reg_lock); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); -+ -+/** -+ * @brief Release the per-context region list lock -+ */ -+void kbase_gpu_vm_unlock(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ mutex_unlock(&kctx->reg_lock); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); -+ -+#ifdef CONFIG_DEBUG_FS -+struct kbase_jit_debugfs_data { -+ int (*func)(struct kbase_jit_debugfs_data *); -+ struct mutex lock; -+ struct kbase_context *kctx; -+ u64 active_value; -+ u64 pool_value; -+ u64 destroy_value; -+ char buffer[50]; -+}; -+ -+static int kbase_jit_debugfs_common_open(struct inode *inode, -+ struct file *file, int (*func)(struct kbase_jit_debugfs_data *)) -+{ -+ struct kbase_jit_debugfs_data *data; -+ -+ data = kzalloc(sizeof(*data), GFP_KERNEL); -+ if (!data) -+ return -ENOMEM; -+ -+ data->func = func; -+ mutex_init(&data->lock); -+ data->kctx = (struct kbase_context *) inode->i_private; -+ -+ file->private_data = data; -+ -+ return nonseekable_open(inode, file); -+} -+ -+static ssize_t kbase_jit_debugfs_common_read(struct file *file, -+ char __user *buf, size_t len, loff_t *ppos) -+{ -+ struct kbase_jit_debugfs_data *data; -+ size_t size; -+ int ret; -+ -+ data = (struct kbase_jit_debugfs_data *) file->private_data; -+ mutex_lock(&data->lock); -+ -+ if (*ppos) { -+ size = strnlen(data->buffer, sizeof(data->buffer)); -+ } else { -+ if (!data->func) { -+ ret = -EACCES; -+ goto out_unlock; -+ } -+ -+ if (data->func(data)) { -+ ret = -EACCES; -+ goto out_unlock; -+ } -+ -+ size = scnprintf(data->buffer, sizeof(data->buffer), -+ "%llu,%llu,%llu", data->active_value, -+ data->pool_value, data->destroy_value); -+ } -+ -+ ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); -+ -+out_unlock: -+ mutex_unlock(&data->lock); -+ return ret; -+} -+ -+static int kbase_jit_debugfs_common_release(struct inode *inode, -+ struct file *file) -+{ -+ kfree(file->private_data); -+ return 0; -+} -+ -+#define KBASE_JIT_DEBUGFS_DECLARE(__fops, __func) \ -+static int __fops ## _open(struct inode *inode, struct file *file) \ -+{ \ -+ return kbase_jit_debugfs_common_open(inode, file, __func); \ -+} \ -+static const struct file_operations __fops = { \ -+ .owner = THIS_MODULE, \ -+ .open = __fops ## _open, \ -+ .release = kbase_jit_debugfs_common_release, \ -+ .read = kbase_jit_debugfs_common_read, \ -+ .write = NULL, \ -+ .llseek = generic_file_llseek, \ -+} -+ -+static int kbase_jit_debugfs_count_get(struct kbase_jit_debugfs_data *data) -+{ -+ struct kbase_context *kctx = data->kctx; -+ struct list_head *tmp; -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ list_for_each(tmp, &kctx->jit_active_head) { -+ data->active_value++; -+ } -+ -+ list_for_each(tmp, &kctx->jit_pool_head) { -+ data->pool_value++; -+ } -+ -+ list_for_each(tmp, &kctx->jit_destroy_head) { -+ data->destroy_value++; -+ } -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ return 0; -+} -+KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_count_fops, -+ kbase_jit_debugfs_count_get); -+ -+static int kbase_jit_debugfs_vm_get(struct kbase_jit_debugfs_data *data) -+{ -+ struct kbase_context *kctx = data->kctx; -+ struct kbase_va_region *reg; -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { -+ data->active_value += reg->nr_pages; -+ } -+ -+ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { -+ data->pool_value += reg->nr_pages; -+ } -+ -+ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { -+ data->destroy_value += reg->nr_pages; -+ } -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ return 0; -+} -+KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_vm_fops, -+ kbase_jit_debugfs_vm_get); -+ -+static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) -+{ -+ struct kbase_context *kctx = data->kctx; -+ struct kbase_va_region *reg; -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { -+ data->active_value += reg->gpu_alloc->nents; -+ } -+ -+ list_for_each_entry(reg, &kctx->jit_pool_head, jit_node) { -+ data->pool_value += reg->gpu_alloc->nents; -+ } -+ -+ list_for_each_entry(reg, &kctx->jit_destroy_head, jit_node) { -+ data->destroy_value += reg->gpu_alloc->nents; -+ } -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ return 0; -+} -+KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, -+ kbase_jit_debugfs_phys_get); -+ -+void kbase_jit_debugfs_init(struct kbase_context *kctx) -+{ -+ /* Debugfs entry for getting the number of JIT allocations. */ -+ debugfs_create_file("mem_jit_count", S_IRUGO, kctx->kctx_dentry, -+ kctx, &kbase_jit_debugfs_count_fops); -+ -+ /* -+ * Debugfs entry for getting the total number of virtual pages -+ * used by JIT allocations. -+ */ -+ debugfs_create_file("mem_jit_vm", S_IRUGO, kctx->kctx_dentry, -+ kctx, &kbase_jit_debugfs_vm_fops); -+ -+ /* -+ * Debugfs entry for getting the number of physical pages used -+ * by JIT allocations. -+ */ -+ debugfs_create_file("mem_jit_phys", S_IRUGO, kctx->kctx_dentry, -+ kctx, &kbase_jit_debugfs_phys_fops); -+} -+#endif /* CONFIG_DEBUG_FS */ -+ -+/** -+ * kbase_jit_destroy_worker - Deferred worker which frees JIT allocations -+ * @work: Work item -+ * -+ * This function does the work of freeing JIT allocations whose physical -+ * backing has been released. -+ */ -+static void kbase_jit_destroy_worker(struct work_struct *work) -+{ -+ struct kbase_context *kctx; -+ struct kbase_va_region *reg; -+ -+ kctx = container_of(work, struct kbase_context, jit_work); -+ do { -+ mutex_lock(&kctx->jit_evict_lock); -+ if (list_empty(&kctx->jit_destroy_head)) { -+ mutex_unlock(&kctx->jit_evict_lock); -+ break; -+ } -+ -+ reg = list_first_entry(&kctx->jit_destroy_head, -+ struct kbase_va_region, jit_node); -+ -+ list_del(®->jit_node); -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ kbase_gpu_vm_lock(kctx); -+ kbase_mem_free_region(kctx, reg); -+ kbase_gpu_vm_unlock(kctx); -+ } while (1); -+} -+ -+int kbase_jit_init(struct kbase_context *kctx) -+{ -+ INIT_LIST_HEAD(&kctx->jit_active_head); -+ INIT_LIST_HEAD(&kctx->jit_pool_head); -+ INIT_LIST_HEAD(&kctx->jit_destroy_head); -+ INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); -+ -+ INIT_LIST_HEAD(&kctx->jit_pending_alloc); -+ INIT_LIST_HEAD(&kctx->jit_atoms_head); -+ -+ return 0; -+} -+ -+struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, -+ struct base_jit_alloc_info *info) -+{ -+ struct kbase_va_region *reg = NULL; -+ struct kbase_va_region *walker; -+ struct kbase_va_region *temp; -+ size_t current_diff = SIZE_MAX; -+ -+ int ret; -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ /* -+ * Scan the pool for an existing allocation which meets our -+ * requirements and remove it. -+ */ -+ list_for_each_entry_safe(walker, temp, &kctx->jit_pool_head, jit_node) { -+ -+ if (walker->nr_pages >= info->va_pages) { -+ size_t min_size, max_size, diff; -+ -+ /* -+ * The JIT allocations VA requirements have been -+ * meet, it's suitable but other allocations -+ * might be a better fit. -+ */ -+ min_size = min_t(size_t, walker->gpu_alloc->nents, -+ info->commit_pages); -+ max_size = max_t(size_t, walker->gpu_alloc->nents, -+ info->commit_pages); -+ diff = max_size - min_size; -+ -+ if (current_diff > diff) { -+ current_diff = diff; -+ reg = walker; -+ } -+ -+ /* The allocation is an exact match, stop looking */ -+ if (current_diff == 0) -+ break; -+ } -+ } -+ -+ if (reg) { -+ /* -+ * Remove the found region from the pool and add it to the -+ * active list. -+ */ -+ list_move(®->jit_node, &kctx->jit_active_head); -+ -+ /* -+ * Remove the allocation from the eviction list as it's no -+ * longer eligible for eviction. This must be done before -+ * dropping the jit_evict_lock -+ */ -+ list_del_init(®->gpu_alloc->evict_node); -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ /* Make the physical backing no longer reclaimable */ -+ if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) -+ goto update_failed; -+ -+ /* Grow the backing if required */ -+ if (reg->gpu_alloc->nents < info->commit_pages) { -+ size_t delta; -+ size_t old_size = reg->gpu_alloc->nents; -+ -+ /* Allocate some more pages */ -+ delta = info->commit_pages - reg->gpu_alloc->nents; -+ if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) -+ != 0) -+ goto update_failed; -+ -+ if (reg->cpu_alloc != reg->gpu_alloc) { -+ if (kbase_alloc_phy_pages_helper( -+ reg->cpu_alloc, delta) != 0) { -+ kbase_free_phy_pages_helper( -+ reg->gpu_alloc, delta); -+ goto update_failed; -+ } -+ } -+ -+ ret = kbase_mem_grow_gpu_mapping(kctx, reg, -+ info->commit_pages, old_size); -+ /* -+ * The grow failed so put the allocation back in the -+ * pool and return failure. -+ */ -+ if (ret) -+ goto update_failed; -+ } -+ kbase_gpu_vm_unlock(kctx); -+ } else { -+ /* No suitable JIT allocation was found so create a new one */ -+ u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | -+ BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | -+ BASE_MEM_COHERENT_LOCAL; -+ u64 gpu_addr; -+ -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, -+ info->extent, &flags, &gpu_addr); -+ if (!reg) -+ goto out_unlocked; -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ list_add(®->jit_node, &kctx->jit_active_head); -+ mutex_unlock(&kctx->jit_evict_lock); -+ } -+ -+ return reg; -+ -+update_failed: -+ /* -+ * An update to an allocation from the pool failed, chances -+ * are slim a new allocation would fair any better so return -+ * the allocation to the pool and return the function with failure. -+ */ -+ kbase_gpu_vm_unlock(kctx); -+ mutex_lock(&kctx->jit_evict_lock); -+ list_move(®->jit_node, &kctx->jit_pool_head); -+ mutex_unlock(&kctx->jit_evict_lock); -+out_unlocked: -+ return NULL; -+} -+ -+void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) -+{ -+ /* The physical backing of memory in the pool is always reclaimable */ -+ kbase_gpu_vm_lock(kctx); -+ kbase_mem_evictable_make(reg->gpu_alloc); -+ kbase_gpu_vm_unlock(kctx); -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ list_move(®->jit_node, &kctx->jit_pool_head); -+ mutex_unlock(&kctx->jit_evict_lock); -+} -+ -+void kbase_jit_backing_lost(struct kbase_va_region *reg) -+{ -+ struct kbase_context *kctx = reg->kctx; -+ -+ lockdep_assert_held(&kctx->jit_evict_lock); -+ -+ /* -+ * JIT allocations will always be on a list, if the region -+ * is not on a list then it's not a JIT allocation. -+ */ -+ if (list_empty(®->jit_node)) -+ return; -+ -+ /* -+ * Freeing the allocation requires locks we might not be able -+ * to take now, so move the allocation to the free list and kick -+ * the worker which will do the freeing. -+ */ -+ list_move(®->jit_node, &kctx->jit_destroy_head); -+ -+ schedule_work(&kctx->jit_work); -+} -+ -+bool kbase_jit_evict(struct kbase_context *kctx) -+{ -+ struct kbase_va_region *reg = NULL; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* Free the oldest allocation from the pool */ -+ mutex_lock(&kctx->jit_evict_lock); -+ if (!list_empty(&kctx->jit_pool_head)) { -+ reg = list_entry(kctx->jit_pool_head.prev, -+ struct kbase_va_region, jit_node); -+ list_del(®->jit_node); -+ } -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ if (reg) -+ kbase_mem_free_region(kctx, reg); -+ -+ return (reg != NULL); -+} -+ -+void kbase_jit_term(struct kbase_context *kctx) -+{ -+ struct kbase_va_region *walker; -+ -+ /* Free all allocations for this context */ -+ -+ /* -+ * Flush the freeing of allocations whose backing has been freed -+ * (i.e. everything in jit_destroy_head). -+ */ -+ cancel_work_sync(&kctx->jit_work); -+ -+ kbase_gpu_vm_lock(kctx); -+ mutex_lock(&kctx->jit_evict_lock); -+ /* Free all allocations from the pool */ -+ while (!list_empty(&kctx->jit_pool_head)) { -+ walker = list_first_entry(&kctx->jit_pool_head, -+ struct kbase_va_region, jit_node); -+ list_del(&walker->jit_node); -+ mutex_unlock(&kctx->jit_evict_lock); -+ kbase_mem_free_region(kctx, walker); -+ mutex_lock(&kctx->jit_evict_lock); -+ } -+ -+ /* Free all allocations from active list */ -+ while (!list_empty(&kctx->jit_active_head)) { -+ walker = list_first_entry(&kctx->jit_active_head, -+ struct kbase_va_region, jit_node); -+ list_del(&walker->jit_node); -+ mutex_unlock(&kctx->jit_evict_lock); -+ kbase_mem_free_region(kctx, walker); -+ mutex_lock(&kctx->jit_evict_lock); -+ } -+ mutex_unlock(&kctx->jit_evict_lock); -+ kbase_gpu_vm_unlock(kctx); -+} -+ -+static int kbase_jd_user_buf_map(struct kbase_context *kctx, -+ struct kbase_va_region *reg) -+{ -+ long pinned_pages; -+ struct kbase_mem_phy_alloc *alloc; -+ struct page **pages; -+ phys_addr_t *pa; -+ long i; -+ int err = -ENOMEM; -+ unsigned long address; -+ struct mm_struct *mm; -+ struct device *dev; -+ unsigned long offset; -+ unsigned long local_size; -+ -+ alloc = reg->gpu_alloc; -+ pa = kbase_get_gpu_phy_pages(reg); -+ address = alloc->imported.user_buf.address; -+ mm = alloc->imported.user_buf.mm; -+ -+ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); -+ -+ pages = alloc->imported.user_buf.pages; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ pinned_pages = get_user_pages(NULL, mm, -+ address, -+ alloc->imported.user_buf.nr_pages, -+ reg->flags & KBASE_REG_GPU_WR, -+ 0, pages, NULL); -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ pinned_pages = get_user_pages_remote(NULL, mm, -+ address, -+ alloc->imported.user_buf.nr_pages, -+ reg->flags & KBASE_REG_GPU_WR, -+ 0, pages, NULL); -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+ pinned_pages = get_user_pages_remote(NULL, mm, -+ address, -+ alloc->imported.user_buf.nr_pages, -+ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -+ pages, NULL); -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) -+ pinned_pages = get_user_pages_remote(NULL, mm, -+ address, -+ alloc->imported.user_buf.nr_pages, -+ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -+ pages, NULL, NULL); -+#else -+ pinned_pages = get_user_pages_remote(mm, -+ address, -+ alloc->imported.user_buf.nr_pages, -+ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -+ pages, NULL, NULL); -+#endif -+ -+ if (pinned_pages <= 0) -+ return pinned_pages; -+ -+ if (pinned_pages != alloc->imported.user_buf.nr_pages) { -+ for (i = 0; i < pinned_pages; i++) -+ put_page(pages[i]); -+ return -ENOMEM; -+ } -+ -+ dev = kctx->kbdev->dev; -+ offset = address & ~PAGE_MASK; -+ local_size = alloc->imported.user_buf.size; -+ -+ for (i = 0; i < pinned_pages; i++) { -+ dma_addr_t dma_addr; -+ unsigned long min; -+ -+ min = MIN(PAGE_SIZE - offset, local_size); -+ dma_addr = dma_map_page(dev, pages[i], -+ offset, min, -+ DMA_BIDIRECTIONAL); -+ if (dma_mapping_error(dev, dma_addr)) -+ goto unwind; -+ -+ alloc->imported.user_buf.dma_addrs[i] = dma_addr; -+ pa[i] = page_to_phys(pages[i]); -+ -+ local_size -= min; -+ offset = 0; -+ } -+ -+ alloc->nents = pinned_pages; -+ -+ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, pa, -+ kbase_reg_current_backed_size(reg), -+ reg->flags); -+ if (err == 0) -+ return 0; -+ -+ alloc->nents = 0; -+ /* fall down */ -+unwind: -+ while (i--) { -+ dma_unmap_page(kctx->kbdev->dev, -+ alloc->imported.user_buf.dma_addrs[i], -+ PAGE_SIZE, DMA_BIDIRECTIONAL); -+ put_page(pages[i]); -+ pages[i] = NULL; -+ } -+ -+ return err; -+} -+ -+static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc, bool writeable) -+{ -+ long i; -+ struct page **pages; -+ unsigned long size = alloc->imported.user_buf.size; -+ -+ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); -+ pages = alloc->imported.user_buf.pages; -+ for (i = 0; i < alloc->imported.user_buf.nr_pages; i++) { -+ unsigned long local_size; -+ dma_addr_t dma_addr = alloc->imported.user_buf.dma_addrs[i]; -+ -+ local_size = MIN(size, PAGE_SIZE - (dma_addr & ~PAGE_MASK)); -+ dma_unmap_page(kctx->kbdev->dev, dma_addr, local_size, -+ DMA_BIDIRECTIONAL); -+ if (writeable) -+ set_page_dirty_lock(pages[i]); -+ put_page(pages[i]); -+ pages[i] = NULL; -+ -+ size -= local_size; -+ } -+ alloc->nents = 0; -+} -+ -+ -+/* to replace sg_dma_len. */ -+#define MALI_SG_DMA_LEN(sg) ((sg)->length) -+ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+static int kbase_jd_umm_map(struct kbase_context *kctx, -+ struct kbase_va_region *reg) -+{ -+ struct sg_table *sgt; -+ struct scatterlist *s; -+ int i; -+ phys_addr_t *pa; -+ int err; -+ size_t count = 0; -+ struct kbase_mem_phy_alloc *alloc; -+ -+ alloc = reg->gpu_alloc; -+ -+ KBASE_DEBUG_ASSERT(alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM); -+ KBASE_DEBUG_ASSERT(NULL == alloc->imported.umm.sgt); -+ sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, -+ DMA_BIDIRECTIONAL); -+ -+ if (IS_ERR_OR_NULL(sgt)) -+ return -EINVAL; -+ -+ /* save for later */ -+ alloc->imported.umm.sgt = sgt; -+ -+ pa = kbase_get_gpu_phy_pages(reg); -+ KBASE_DEBUG_ASSERT(pa); -+ -+ for_each_sg(sgt->sgl, s, sgt->nents, i) { -+ int j; -+ size_t pages = PFN_UP(MALI_SG_DMA_LEN(s)); -+ -+ WARN_ONCE(MALI_SG_DMA_LEN(s) & (PAGE_SIZE-1), -+ "MALI_SG_DMA_LEN(s)=%u is not a multiple of PAGE_SIZE\n", -+ MALI_SG_DMA_LEN(s)); -+ -+ WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), -+ "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", -+ (unsigned long long) sg_dma_address(s)); -+ -+ for (j = 0; (j < pages) && (count < reg->nr_pages); j++, -+ count++) -+ *pa++ = sg_dma_address(s) + (j << PAGE_SHIFT); -+ WARN_ONCE(j < pages, -+ "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", -+ alloc->imported.umm.dma_buf->size); -+ } -+ -+ if (!(reg->flags & KBASE_REG_IMPORT_PAD) && -+ WARN_ONCE(count < reg->nr_pages, -+ "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", -+ alloc->imported.umm.dma_buf->size)) { -+ err = -EINVAL; -+ goto err_unmap_attachment; -+ } -+ -+ /* Update nents as we now have pages to map */ -+ alloc->nents = reg->nr_pages; -+ -+ err = kbase_mmu_insert_pages(kctx, reg->start_pfn, -+ kbase_get_gpu_phy_pages(reg), -+ count, -+ reg->flags | KBASE_REG_GPU_WR | KBASE_REG_GPU_RD); -+ if (err) -+ goto err_unmap_attachment; -+ -+ if (reg->flags & KBASE_REG_IMPORT_PAD) { -+ err = kbase_mmu_insert_single_page(kctx, -+ reg->start_pfn + count, -+ page_to_phys(kctx->aliasing_sink_page), -+ reg->nr_pages - count, -+ (reg->flags | KBASE_REG_GPU_RD) & -+ ~KBASE_REG_GPU_WR); -+ if (err) -+ goto err_teardown_orig_pages; -+ } -+ -+ return 0; -+ -+err_teardown_orig_pages: -+ kbase_mmu_teardown_pages(kctx, reg->start_pfn, count); -+err_unmap_attachment: -+ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, -+ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); -+ alloc->imported.umm.sgt = NULL; -+ -+ return err; -+} -+ -+static void kbase_jd_umm_unmap(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc) -+{ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(alloc); -+ KBASE_DEBUG_ASSERT(alloc->imported.umm.dma_attachment); -+ KBASE_DEBUG_ASSERT(alloc->imported.umm.sgt); -+ dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, -+ alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); -+ alloc->imported.umm.sgt = NULL; -+ alloc->nents = 0; -+} -+#endif /* CONFIG_DMA_SHARED_BUFFER */ -+ -+#if (defined(CONFIG_KDS) && defined(CONFIG_UMP)) \ -+ || defined(CONFIG_DMA_SHARED_BUFFER_USES_KDS) -+static void add_kds_resource(struct kds_resource *kds_res, -+ struct kds_resource **kds_resources, u32 *kds_res_count, -+ unsigned long *kds_access_bitmap, bool exclusive) -+{ -+ u32 i; -+ -+ for (i = 0; i < *kds_res_count; i++) { -+ /* Duplicate resource, ignore */ -+ if (kds_resources[i] == kds_res) -+ return; -+ } -+ -+ kds_resources[*kds_res_count] = kds_res; -+ if (exclusive) -+ set_bit(*kds_res_count, kds_access_bitmap); -+ (*kds_res_count)++; -+} -+#endif -+ -+struct kbase_mem_phy_alloc *kbase_map_external_resource( -+ struct kbase_context *kctx, struct kbase_va_region *reg, -+ struct mm_struct *locked_mm -+#ifdef CONFIG_KDS -+ , u32 *kds_res_count, struct kds_resource **kds_resources, -+ unsigned long *kds_access_bitmap, bool exclusive -+#endif -+ ) -+{ -+ int err; -+ -+ /* decide what needs to happen for this resource */ -+ switch (reg->gpu_alloc->type) { -+ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { -+ if (reg->gpu_alloc->imported.user_buf.mm != locked_mm) -+ goto exit; -+ -+ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; -+ if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { -+ err = kbase_jd_user_buf_map(kctx, reg); -+ if (err) { -+ reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; -+ goto exit; -+ } -+ } -+ } -+ break; -+ case KBASE_MEM_TYPE_IMPORTED_UMP: { -+#if defined(CONFIG_KDS) && defined(CONFIG_UMP) -+ if (kds_res_count) { -+ struct kds_resource *kds_res; -+ -+ kds_res = ump_dd_kds_resource_get( -+ reg->gpu_alloc->imported.ump_handle); -+ if (kds_res) -+ add_kds_resource(kds_res, kds_resources, -+ kds_res_count, -+ kds_access_bitmap, exclusive); -+ } -+#endif /*defined(CONFIG_KDS) && defined(CONFIG_UMP) */ -+ break; -+ } -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ case KBASE_MEM_TYPE_IMPORTED_UMM: { -+#ifdef CONFIG_DMA_SHARED_BUFFER_USES_KDS -+ if (kds_res_count) { -+ struct kds_resource *kds_res; -+ -+ kds_res = get_dma_buf_kds_resource( -+ reg->gpu_alloc->imported.umm.dma_buf); -+ if (kds_res) -+ add_kds_resource(kds_res, kds_resources, -+ kds_res_count, -+ kds_access_bitmap, exclusive); -+ } -+#endif -+ reg->gpu_alloc->imported.umm.current_mapping_usage_count++; -+ if (1 == reg->gpu_alloc->imported.umm.current_mapping_usage_count) { -+ err = kbase_jd_umm_map(kctx, reg); -+ if (err) { -+ reg->gpu_alloc->imported.umm.current_mapping_usage_count--; -+ goto exit; -+ } -+ } -+ break; -+ } -+#endif -+ default: -+ goto exit; -+ } -+ -+ return kbase_mem_phy_alloc_get(reg->gpu_alloc); -+exit: -+ return NULL; -+} -+ -+void kbase_unmap_external_resource(struct kbase_context *kctx, -+ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) -+{ -+ switch (alloc->type) { -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ case KBASE_MEM_TYPE_IMPORTED_UMM: { -+ alloc->imported.umm.current_mapping_usage_count--; -+ -+ if (0 == alloc->imported.umm.current_mapping_usage_count) { -+ if (reg && reg->gpu_alloc == alloc) { -+ int err; -+ -+ err = kbase_mmu_teardown_pages( -+ kctx, -+ reg->start_pfn, -+ alloc->nents); -+ WARN_ON(err); -+ } -+ -+ kbase_jd_umm_unmap(kctx, alloc); -+ } -+ } -+ break; -+#endif /* CONFIG_DMA_SHARED_BUFFER */ -+ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { -+ alloc->imported.user_buf.current_mapping_usage_count--; -+ -+ if (0 == alloc->imported.user_buf.current_mapping_usage_count) { -+ bool writeable = true; -+ -+ if (reg && reg->gpu_alloc == alloc) -+ kbase_mmu_teardown_pages( -+ kctx, -+ reg->start_pfn, -+ kbase_reg_current_backed_size(reg)); -+ -+ if (reg && ((reg->flags & KBASE_REG_GPU_WR) == 0)) -+ writeable = false; -+ -+ kbase_jd_user_buf_unmap(kctx, alloc, writeable); -+ } -+ } -+ break; -+ default: -+ break; -+ } -+ kbase_mem_phy_alloc_put(alloc); -+} -+ -+struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( -+ struct kbase_context *kctx, u64 gpu_addr) -+{ -+ struct kbase_ctx_ext_res_meta *meta = NULL; -+ struct kbase_ctx_ext_res_meta *walker; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* -+ * Walk the per context external resource metadata list for the -+ * metadata which matches the region which is being acquired. -+ */ -+ list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { -+ if (walker->gpu_addr == gpu_addr) { -+ meta = walker; -+ break; -+ } -+ } -+ -+ /* No metadata exists so create one. */ -+ if (!meta) { -+ struct kbase_va_region *reg; -+ -+ /* Find the region */ -+ reg = kbase_region_tracker_find_region_enclosing_address( -+ kctx, gpu_addr); -+ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) -+ goto failed; -+ -+ /* Allocate the metadata object */ -+ meta = kzalloc(sizeof(*meta), GFP_KERNEL); -+ if (!meta) -+ goto failed; -+ -+ /* -+ * Fill in the metadata object and acquire a reference -+ * for the physical resource. -+ */ -+ meta->alloc = kbase_map_external_resource(kctx, reg, NULL -+#ifdef CONFIG_KDS -+ , NULL, NULL, -+ NULL, false -+#endif -+ ); -+ -+ if (!meta->alloc) -+ goto fail_map; -+ -+ meta->gpu_addr = reg->start_pfn << PAGE_SHIFT; -+ -+ list_add(&meta->ext_res_node, &kctx->ext_res_meta_head); -+ } -+ -+ return meta; -+ -+fail_map: -+ kfree(meta); -+failed: -+ return NULL; -+} -+ -+bool kbase_sticky_resource_release(struct kbase_context *kctx, -+ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) -+{ -+ struct kbase_ctx_ext_res_meta *walker; -+ struct kbase_va_region *reg; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* Search of the metadata if one isn't provided. */ -+ if (!meta) { -+ /* -+ * Walk the per context external resource metadata list for the -+ * metadata which matches the region which is being released. -+ */ -+ list_for_each_entry(walker, &kctx->ext_res_meta_head, -+ ext_res_node) { -+ if (walker->gpu_addr == gpu_addr) { -+ meta = walker; -+ break; -+ } -+ } -+ } -+ -+ /* No metadata so just return. */ -+ if (!meta) -+ return false; -+ -+ /* Drop the physical memory reference and free the metadata. */ -+ reg = kbase_region_tracker_find_region_enclosing_address( -+ kctx, -+ meta->gpu_addr); -+ -+ kbase_unmap_external_resource(kctx, reg, meta->alloc); -+ list_del(&meta->ext_res_node); -+ kfree(meta); -+ -+ return true; -+} -+ -+int kbase_sticky_resource_init(struct kbase_context *kctx) -+{ -+ INIT_LIST_HEAD(&kctx->ext_res_meta_head); -+ -+ return 0; -+} -+ -+void kbase_sticky_resource_term(struct kbase_context *kctx) -+{ -+ struct kbase_ctx_ext_res_meta *walker; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* -+ * Free any sticky resources which haven't been unmapped. -+ * -+ * Note: -+ * We don't care about refcounts at this point as no future -+ * references to the meta data will be made. -+ * Region termination would find these if we didn't free them -+ * here, but it's more efficient if we do the clean up here. -+ */ -+ while (!list_empty(&kctx->ext_res_meta_head)) { -+ walker = list_first_entry(&kctx->ext_res_meta_head, -+ struct kbase_ctx_ext_res_meta, ext_res_node); -+ -+ kbase_sticky_resource_release(kctx, walker, 0); -+ } -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem.h b/drivers/gpu/arm/midgard/mali_kbase_mem.h -new file mode 100755 -index 000000000..3f3eaa3fd ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem.h -@@ -0,0 +1,1068 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_mem.h -+ * Base kernel memory APIs -+ */ -+ -+#ifndef _KBASE_MEM_H_ -+#define _KBASE_MEM_H_ -+ -+#ifndef _KBASE_H_ -+#error "Don't include this file directly, use mali_kbase.h instead" -+#endif -+ -+#include -+#ifdef CONFIG_KDS -+#include -+#endif /* CONFIG_KDS */ -+#ifdef CONFIG_UMP -+#include -+#endif /* CONFIG_UMP */ -+#include "mali_base_kernel.h" -+#include -+#include "mali_kbase_pm.h" -+#include "mali_kbase_defs.h" -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+#include "mali_kbase_gator.h" -+#endif -+/* Required for kbase_mem_evictable_unmake */ -+#include "mali_kbase_mem_linux.h" -+ -+/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ -+#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ -+ -+/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. -+The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and -+page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table -+updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ -+#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ -+ -+#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ -+ -+/* This must always be a power of 2 */ -+#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) -+#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) -+#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) -+/** -+ * A CPU mapping -+ */ -+struct kbase_cpu_mapping { -+ struct list_head mappings_list; -+ struct kbase_mem_phy_alloc *alloc; -+ struct kbase_context *kctx; -+ struct kbase_va_region *region; -+ int count; -+ int free_on_close; -+}; -+ -+enum kbase_memory_type { -+ KBASE_MEM_TYPE_NATIVE, -+ KBASE_MEM_TYPE_IMPORTED_UMP, -+ KBASE_MEM_TYPE_IMPORTED_UMM, -+ KBASE_MEM_TYPE_IMPORTED_USER_BUF, -+ KBASE_MEM_TYPE_ALIAS, -+ KBASE_MEM_TYPE_TB, -+ KBASE_MEM_TYPE_RAW -+}; -+ -+/* internal structure, mirroring base_mem_aliasing_info, -+ * but with alloc instead of a gpu va (handle) */ -+struct kbase_aliased { -+ struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ -+ u64 offset; /* in pages */ -+ u64 length; /* in pages */ -+}; -+ -+/** -+ * @brief Physical pages tracking object properties -+ */ -+#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1ul << 0) -+#define KBASE_MEM_PHY_ALLOC_LARGE (1ul << 1) -+ -+/* physical pages tracking object. -+ * Set up to track N pages. -+ * N not stored here, the creator holds that info. -+ * This object only tracks how many elements are actually valid (present). -+ * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc is not -+ * shared with another region or client. CPU mappings are OK to exist when changing, as -+ * long as the tracked mappings objects are updated as part of the change. -+ */ -+struct kbase_mem_phy_alloc { -+ struct kref kref; /* number of users of this alloc */ -+ atomic_t gpu_mappings; -+ size_t nents; /* 0..N */ -+ phys_addr_t *pages; /* N elements, only 0..nents are valid */ -+ -+ /* kbase_cpu_mappings */ -+ struct list_head mappings; -+ -+ /* Node used to store this allocation on the eviction list */ -+ struct list_head evict_node; -+ /* Physical backing size when the pages where evicted */ -+ size_t evicted; -+ /* -+ * Back reference to the region structure which created this -+ * allocation, or NULL if it has been freed. -+ */ -+ struct kbase_va_region *reg; -+ -+ /* type of buffer */ -+ enum kbase_memory_type type; -+ -+ unsigned long properties; -+ -+ /* member in union valid based on @a type */ -+ union { -+#ifdef CONFIG_UMP -+ ump_dd_handle ump_handle; -+#endif /* CONFIG_UMP */ -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+ struct { -+ struct dma_buf *dma_buf; -+ struct dma_buf_attachment *dma_attachment; -+ unsigned int current_mapping_usage_count; -+ struct sg_table *sgt; -+ } umm; -+#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ -+ struct { -+ u64 stride; -+ size_t nents; -+ struct kbase_aliased *aliased; -+ } alias; -+ /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */ -+ struct kbase_context *kctx; -+ struct kbase_alloc_import_user_buf { -+ unsigned long address; -+ unsigned long size; -+ unsigned long nr_pages; -+ struct page **pages; -+ /* top bit (1<<31) of current_mapping_usage_count -+ * specifies that this import was pinned on import -+ * See PINNED_ON_IMPORT -+ */ -+ u32 current_mapping_usage_count; -+ struct mm_struct *mm; -+ dma_addr_t *dma_addrs; -+ } user_buf; -+ } imported; -+}; -+ -+/* The top bit of kbase_alloc_import_user_buf::current_mapping_usage_count is -+ * used to signify that a buffer was pinned when it was imported. Since the -+ * reference count is limited by the number of atoms that can be submitted at -+ * once there should be no danger of overflowing into this bit. -+ * Stealing the top bit also has the benefit that -+ * current_mapping_usage_count != 0 if and only if the buffer is mapped. -+ */ -+#define PINNED_ON_IMPORT (1<<31) -+ -+static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) -+{ -+ KBASE_DEBUG_ASSERT(alloc); -+ /* we only track mappings of NATIVE buffers */ -+ if (alloc->type == KBASE_MEM_TYPE_NATIVE) -+ atomic_inc(&alloc->gpu_mappings); -+} -+ -+static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc) -+{ -+ KBASE_DEBUG_ASSERT(alloc); -+ /* we only track mappings of NATIVE buffers */ -+ if (alloc->type == KBASE_MEM_TYPE_NATIVE) -+ if (0 > atomic_dec_return(&alloc->gpu_mappings)) { -+ pr_err("Mismatched %s:\n", __func__); -+ dump_stack(); -+ } -+} -+ -+void kbase_mem_kref_free(struct kref *kref); -+ -+int kbase_mem_init(struct kbase_device *kbdev); -+void kbase_mem_halt(struct kbase_device *kbdev); -+void kbase_mem_term(struct kbase_device *kbdev); -+ -+static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc) -+{ -+ kref_get(&alloc->kref); -+ return alloc; -+} -+ -+static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc) -+{ -+ kref_put(&alloc->kref, kbase_mem_kref_free); -+ return NULL; -+} -+ -+/** -+ * A GPU memory region, and attributes for CPU mappings. -+ */ -+struct kbase_va_region { -+ struct rb_node rblink; -+ struct list_head link; -+ -+ struct kbase_context *kctx; /* Backlink to base context */ -+ -+ u64 start_pfn; /* The PFN in GPU space */ -+ size_t nr_pages; -+ -+/* Free region */ -+#define KBASE_REG_FREE (1ul << 0) -+/* CPU write access */ -+#define KBASE_REG_CPU_WR (1ul << 1) -+/* GPU write access */ -+#define KBASE_REG_GPU_WR (1ul << 2) -+/* No eXecute flag */ -+#define KBASE_REG_GPU_NX (1ul << 3) -+/* Is CPU cached? */ -+#define KBASE_REG_CPU_CACHED (1ul << 4) -+/* Is GPU cached? */ -+#define KBASE_REG_GPU_CACHED (1ul << 5) -+ -+#define KBASE_REG_GROWABLE (1ul << 6) -+/* Can grow on pf? */ -+#define KBASE_REG_PF_GROW (1ul << 7) -+ -+/* VA managed by us */ -+#define KBASE_REG_CUSTOM_VA (1ul << 8) -+ -+/* inner shareable coherency */ -+#define KBASE_REG_SHARE_IN (1ul << 9) -+/* inner & outer shareable coherency */ -+#define KBASE_REG_SHARE_BOTH (1ul << 10) -+ -+/* Space for 4 different zones */ -+#define KBASE_REG_ZONE_MASK (3ul << 11) -+#define KBASE_REG_ZONE(x) (((x) & 3) << 11) -+ -+/* GPU read access */ -+#define KBASE_REG_GPU_RD (1ul<<13) -+/* CPU read access */ -+#define KBASE_REG_CPU_RD (1ul<<14) -+ -+/* Index of chosen MEMATTR for this region (0..7) */ -+#define KBASE_REG_MEMATTR_MASK (7ul << 16) -+#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) -+#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) -+ -+#define KBASE_REG_SECURE (1ul << 19) -+ -+#define KBASE_REG_DONT_NEED (1ul << 20) -+ -+/* Imported buffer is padded? */ -+#define KBASE_REG_IMPORT_PAD (1ul << 21) -+ -+#define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) -+ -+/* only used with 32-bit clients */ -+/* -+ * On a 32bit platform, custom VA should be wired from (4GB + shader region) -+ * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface -+ * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference). -+ * So we put the default limit to the maximum possible on Linux and shrink -+ * it down, if required by the GPU, during initialization. -+ */ -+ -+/* -+ * Dedicated 16MB region for shader code: -+ * VA range 0x101000000-0x102000000 -+ */ -+#define KBASE_REG_ZONE_EXEC KBASE_REG_ZONE(1) -+#define KBASE_REG_ZONE_EXEC_BASE (0x101000000ULL >> PAGE_SHIFT) -+#define KBASE_REG_ZONE_EXEC_SIZE ((16ULL * 1024 * 1024) >> PAGE_SHIFT) -+ -+#define KBASE_REG_ZONE_CUSTOM_VA KBASE_REG_ZONE(2) -+#define KBASE_REG_ZONE_CUSTOM_VA_BASE (KBASE_REG_ZONE_EXEC_BASE + KBASE_REG_ZONE_EXEC_SIZE) /* Starting after KBASE_REG_ZONE_EXEC */ -+#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) -+/* end 32-bit clients only */ -+ -+ unsigned long flags; -+ -+ size_t extent; /* nr of pages alloc'd on PF */ -+ -+ struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */ -+ struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */ -+ -+ /* non-NULL if this memory object is a kds_resource */ -+ struct kds_resource *kds_res; -+ -+ /* List head used to store the region in the JIT allocation pool */ -+ struct list_head jit_node; -+}; -+ -+/* Common functions */ -+static inline phys_addr_t *kbase_get_cpu_phy_pages(struct kbase_va_region *reg) -+{ -+ KBASE_DEBUG_ASSERT(reg); -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->gpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); -+ -+ return reg->cpu_alloc->pages; -+} -+ -+static inline phys_addr_t *kbase_get_gpu_phy_pages(struct kbase_va_region *reg) -+{ -+ KBASE_DEBUG_ASSERT(reg); -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->gpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); -+ -+ return reg->gpu_alloc->pages; -+} -+ -+static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg) -+{ -+ KBASE_DEBUG_ASSERT(reg); -+ /* if no alloc object the backed size naturally is 0 */ -+ if (!reg->cpu_alloc) -+ return 0; -+ -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->gpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents); -+ -+ return reg->cpu_alloc->nents; -+} -+ -+#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */ -+ -+static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type) -+{ -+ struct kbase_mem_phy_alloc *alloc; -+ size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages; -+ size_t per_page_size = sizeof(*alloc->pages); -+ -+ /* Imported pages may have page private data already in use */ -+ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { -+ alloc_size += nr_pages * -+ sizeof(*alloc->imported.user_buf.dma_addrs); -+ per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs); -+ } -+ -+ /* -+ * Prevent nr_pages*per_page_size + sizeof(*alloc) from -+ * wrapping around. -+ */ -+ if (nr_pages > ((((size_t) -1) - sizeof(*alloc)) -+ / per_page_size)) -+ return ERR_PTR(-ENOMEM); -+ -+ /* Allocate based on the size to reduce internal fragmentation of vmem */ -+ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) -+ alloc = vzalloc(alloc_size); -+ else -+ alloc = kzalloc(alloc_size, GFP_KERNEL); -+ -+ if (!alloc) -+ return ERR_PTR(-ENOMEM); -+ -+ /* Store allocation method */ -+ if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD) -+ alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE; -+ -+ kref_init(&alloc->kref); -+ atomic_set(&alloc->gpu_mappings, 0); -+ alloc->nents = 0; -+ alloc->pages = (void *)(alloc + 1); -+ INIT_LIST_HEAD(&alloc->mappings); -+ alloc->type = type; -+ -+ if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) -+ alloc->imported.user_buf.dma_addrs = -+ (void *) (alloc->pages + nr_pages); -+ -+ return alloc; -+} -+ -+static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, -+ struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(reg); -+ KBASE_DEBUG_ASSERT(!reg->cpu_alloc); -+ KBASE_DEBUG_ASSERT(!reg->gpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE); -+ -+ reg->cpu_alloc = kbase_alloc_create(reg->nr_pages, -+ KBASE_MEM_TYPE_NATIVE); -+ if (IS_ERR(reg->cpu_alloc)) -+ return PTR_ERR(reg->cpu_alloc); -+ else if (!reg->cpu_alloc) -+ return -ENOMEM; -+ reg->cpu_alloc->imported.kctx = kctx; -+ INIT_LIST_HEAD(®->cpu_alloc->evict_node); -+ if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE) -+ && (reg->flags & KBASE_REG_CPU_CACHED)) { -+ reg->gpu_alloc = kbase_alloc_create(reg->nr_pages, -+ KBASE_MEM_TYPE_NATIVE); -+ reg->gpu_alloc->imported.kctx = kctx; -+ INIT_LIST_HEAD(®->gpu_alloc->evict_node); -+ } else { -+ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); -+ } -+ -+ INIT_LIST_HEAD(®->jit_node); -+ reg->flags &= ~KBASE_REG_FREE; -+ return 0; -+} -+ -+static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages) -+{ -+ int new_val = atomic_add_return(num_pages, used_pages); -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_total_alloc_pages_change((long long int)new_val); -+#endif -+ return new_val; -+} -+ -+static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages) -+{ -+ int new_val = atomic_sub_return(num_pages, used_pages); -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_total_alloc_pages_change((long long int)new_val); -+#endif -+ return new_val; -+} -+ -+/* -+ * Max size for kbdev memory pool (in pages) -+ */ -+#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) -+ -+/* -+ * Max size for kctx memory pool (in pages) -+ */ -+#define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) -+ -+/** -+ * kbase_mem_pool_init - Create a memory pool for a kbase device -+ * @pool: Memory pool to initialize -+ * @max_size: Maximum number of free pages the pool can hold -+ * @kbdev: Kbase device where memory is used -+ * @next_pool: Pointer to the next pool or NULL. -+ * -+ * Allocations from @pool are in whole pages. Each @pool has a free list where -+ * pages can be quickly allocated from. The free list is initially empty and -+ * filled whenever pages are freed back to the pool. The number of free pages -+ * in the pool will in general not exceed @max_size, but the pool may in -+ * certain corner cases grow above @max_size. -+ * -+ * If @next_pool is not NULL, we will allocate from @next_pool before going to -+ * the kernel allocator. Similarily pages can spill over to @next_pool when -+ * @pool is full. Pages are zeroed before they spill over to another pool, to -+ * prevent leaking information between applications. -+ * -+ * A shrinker is registered so that Linux mm can reclaim pages from the pool as -+ * needed. -+ * -+ * Return: 0 on success, negative -errno on error -+ */ -+int kbase_mem_pool_init(struct kbase_mem_pool *pool, -+ size_t max_size, -+ struct kbase_device *kbdev, -+ struct kbase_mem_pool *next_pool); -+ -+/** -+ * kbase_mem_pool_term - Destroy a memory pool -+ * @pool: Memory pool to destroy -+ * -+ * Pages in the pool will spill over to @next_pool (if available) or freed to -+ * the kernel. -+ */ -+void kbase_mem_pool_term(struct kbase_mem_pool *pool); -+ -+/** -+ * kbase_mem_pool_alloc - Allocate a page from memory pool -+ * @pool: Memory pool to allocate from -+ * -+ * Allocations from the pool are made as follows: -+ * 1. If there are free pages in the pool, allocate a page from @pool. -+ * 2. Otherwise, if @next_pool is not NULL and has free pages, allocate a page -+ * from @next_pool. -+ * 3. Return NULL if no memory in the pool -+ * -+ * Return: Pointer to allocated page, or NULL if allocation failed. -+ */ -+struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool); -+ -+/** -+ * kbase_mem_pool_free - Free a page to memory pool -+ * @pool: Memory pool where page should be freed -+ * @page: Page to free to the pool -+ * @dirty: Whether some of the page may be dirty in the cache. -+ * -+ * Pages are freed to the pool as follows: -+ * 1. If @pool is not full, add @page to @pool. -+ * 2. Otherwise, if @next_pool is not NULL and not full, add @page to -+ * @next_pool. -+ * 3. Finally, free @page to the kernel. -+ */ -+void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page, -+ bool dirty); -+ -+/** -+ * kbase_mem_pool_alloc_pages - Allocate pages from memory pool -+ * @pool: Memory pool to allocate from -+ * @nr_pages: Number of pages to allocate -+ * @pages: Pointer to array where the physical address of the allocated -+ * pages will be stored. -+ * -+ * Like kbase_mem_pool_alloc() but optimized for allocating many pages. -+ * -+ * Return: 0 on success, negative -errno on error -+ */ -+int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, -+ phys_addr_t *pages); -+ -+/** -+ * kbase_mem_pool_free_pages - Free pages to memory pool -+ * @pool: Memory pool where pages should be freed -+ * @nr_pages: Number of pages to free -+ * @pages: Pointer to array holding the physical addresses of the pages to -+ * free. -+ * @dirty: Whether any pages may be dirty in the cache. -+ * @reclaimed: Whether the pages where reclaimable and thus should bypass -+ * the pool and go straight to the kernel. -+ * -+ * Like kbase_mem_pool_free() but optimized for freeing many pages. -+ */ -+void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, -+ phys_addr_t *pages, bool dirty, bool reclaimed); -+ -+/** -+ * kbase_mem_pool_size - Get number of free pages in memory pool -+ * @pool: Memory pool to inspect -+ * -+ * Note: the size of the pool may in certain corner cases exceed @max_size! -+ * -+ * Return: Number of free pages in the pool -+ */ -+static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool) -+{ -+ return READ_ONCE(pool->cur_size); -+} -+ -+/** -+ * kbase_mem_pool_max_size - Get maximum number of free pages in memory pool -+ * @pool: Memory pool to inspect -+ * -+ * Return: Maximum number of free pages in the pool -+ */ -+static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool) -+{ -+ return pool->max_size; -+} -+ -+ -+/** -+ * kbase_mem_pool_set_max_size - Set maximum number of free pages in memory pool -+ * @pool: Memory pool to inspect -+ * @max_size: Maximum number of free pages the pool can hold -+ * -+ * If @max_size is reduced, the pool will be shrunk to adhere to the new limit. -+ * For details see kbase_mem_pool_shrink(). -+ */ -+void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); -+ -+/** -+ * kbase_mem_pool_grow - Grow the pool -+ * @pool: Memory pool to grow -+ * @nr_to_grow: Number of pages to add to the pool -+ * -+ * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to -+ * become larger than the maximum size specified. -+ * -+ * Returns: 0 on success, -ENOMEM if unable to allocate sufficent pages -+ */ -+int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow); -+ -+/** -+ * kbase_mem_pool_trim - Grow or shrink the pool to a new size -+ * @pool: Memory pool to trim -+ * @new_size: New number of pages in the pool -+ * -+ * If @new_size > @cur_size, fill the pool with new pages from the kernel, but -+ * not above the max_size for the pool. -+ * If @new_size < @cur_size, shrink the pool by freeing pages to the kernel. -+ */ -+void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size); -+ -+/* -+ * kbase_mem_alloc_page - Allocate a new page for a device -+ * @kbdev: The kbase device -+ * -+ * Most uses should use kbase_mem_pool_alloc to allocate a page. However that -+ * function can fail in the event the pool is empty. -+ * -+ * Return: A new page or NULL if no memory -+ */ -+struct page *kbase_mem_alloc_page(struct kbase_device *kbdev); -+ -+int kbase_region_tracker_init(struct kbase_context *kctx); -+int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages); -+void kbase_region_tracker_term(struct kbase_context *kctx); -+ -+struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr); -+ -+/** -+ * @brief Check that a pointer is actually a valid region. -+ * -+ * Must be called with context lock held. -+ */ -+struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr); -+ -+struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone); -+void kbase_free_alloced_region(struct kbase_va_region *reg); -+int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); -+ -+bool kbase_check_alloc_flags(unsigned long flags); -+bool kbase_check_import_flags(unsigned long flags); -+ -+/** -+ * kbase_update_region_flags - Convert user space flags to kernel region flags -+ * -+ * @kctx: kbase context -+ * @reg: The region to update the flags on -+ * @flags: The flags passed from user space -+ * -+ * The user space flag BASE_MEM_COHERENT_SYSTEM_REQUIRED will be rejected and -+ * this function will fail if the system does not support system coherency. -+ * -+ * Return: 0 if successful, -EINVAL if the flags are not supported -+ */ -+int kbase_update_region_flags(struct kbase_context *kctx, -+ struct kbase_va_region *reg, unsigned long flags); -+ -+void kbase_gpu_vm_lock(struct kbase_context *kctx); -+void kbase_gpu_vm_unlock(struct kbase_context *kctx); -+ -+int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); -+ -+int kbase_mmu_init(struct kbase_context *kctx); -+void kbase_mmu_term(struct kbase_context *kctx); -+ -+phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx); -+void kbase_mmu_free_pgd(struct kbase_context *kctx); -+int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, -+ phys_addr_t *phys, size_t nr, -+ unsigned long flags); -+int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, -+ phys_addr_t *phys, size_t nr, -+ unsigned long flags); -+int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, -+ phys_addr_t phys, size_t nr, -+ unsigned long flags); -+ -+int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr); -+int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags); -+ -+/** -+ * @brief Register region and map it on the GPU. -+ * -+ * Call kbase_add_va_region() and map the region on the GPU. -+ */ -+int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); -+ -+/** -+ * @brief Remove the region from the GPU and unregister it. -+ * -+ * Must be called with context lock held. -+ */ -+int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg); -+ -+/** -+ * The caller has the following locking conditions: -+ * - It must hold kbase_device->mmu_hw_mutex -+ * - It must hold the hwaccess_lock -+ */ -+void kbase_mmu_update(struct kbase_context *kctx); -+ -+/** -+ * kbase_mmu_disable() - Disable the MMU for a previously active kbase context. -+ * @kctx: Kbase context -+ * -+ * Disable and perform the required cache maintenance to remove the all -+ * data from provided kbase context from the GPU caches. -+ * -+ * The caller has the following locking conditions: -+ * - It must hold kbase_device->mmu_hw_mutex -+ * - It must hold the hwaccess_lock -+ */ -+void kbase_mmu_disable(struct kbase_context *kctx); -+ -+/** -+ * kbase_mmu_disable_as() - Set the MMU to unmapped mode for the specified -+ * address space. -+ * @kbdev: Kbase device -+ * @as_nr: The address space number to set to unmapped. -+ * -+ * This function must only be called during reset/power-up and it used to -+ * ensure the registers are in a known state. -+ * -+ * The caller must hold kbdev->mmu_hw_mutex. -+ */ -+void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); -+ -+void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); -+ -+/** Dump the MMU tables to a buffer -+ * -+ * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the -+ * buffer is too small then the return value will be NULL. -+ * -+ * The GPU vm lock must be held when calling this function. -+ * -+ * The buffer returned should be freed with @ref vfree when it is no longer required. -+ * -+ * @param[in] kctx The kbase context to dump -+ * @param[in] nr_pages The number of pages to allocate for the buffer. -+ * -+ * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too -+ * small) -+ */ -+void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); -+ -+/** -+ * kbase_sync_now - Perform cache maintenance on a memory region -+ * -+ * @kctx: The kbase context of the region -+ * @sset: A syncset structure describing the region and direction of the -+ * synchronisation required -+ * -+ * Return: 0 on success or error code -+ */ -+int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset); -+void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa, -+ phys_addr_t gpu_pa, off_t offset, size_t size, -+ enum kbase_sync_type sync_fn); -+void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); -+void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr); -+ -+/* OS specific functions */ -+int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr); -+int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg); -+void kbase_os_mem_map_lock(struct kbase_context *kctx); -+void kbase_os_mem_map_unlock(struct kbase_context *kctx); -+ -+/** -+ * @brief Update the memory allocation counters for the current process -+ * -+ * OS specific call to updates the current memory allocation counters for the current process with -+ * the supplied delta. -+ * -+ * @param[in] kctx The kbase context -+ * @param[in] pages The desired delta to apply to the memory usage counters. -+ */ -+ -+void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); -+ -+/** -+ * @brief Add to the memory allocation counters for the current process -+ * -+ * OS specific call to add to the current memory allocation counters for the current process by -+ * the supplied amount. -+ * -+ * @param[in] kctx The kernel base context used for the allocation. -+ * @param[in] pages The desired delta to apply to the memory usage counters. -+ */ -+ -+static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) -+{ -+ kbasep_os_process_page_usage_update(kctx, pages); -+} -+ -+/** -+ * @brief Subtract from the memory allocation counters for the current process -+ * -+ * OS specific call to subtract from the current memory allocation counters for the current process by -+ * the supplied amount. -+ * -+ * @param[in] kctx The kernel base context used for the allocation. -+ * @param[in] pages The desired delta to apply to the memory usage counters. -+ */ -+ -+static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) -+{ -+ kbasep_os_process_page_usage_update(kctx, 0 - pages); -+} -+ -+/** -+ * kbasep_find_enclosing_cpu_mapping_offset() - Find the offset of the CPU -+ * mapping of a memory allocation containing a given address range -+ * -+ * Searches for a CPU mapping of any part of any region that fully encloses the -+ * CPU virtual address range specified by @uaddr and @size. Returns a failure -+ * indication if only part of the address range lies within a CPU mapping. -+ * -+ * @kctx: The kernel base context used for the allocation. -+ * @uaddr: Start of the CPU virtual address range. -+ * @size: Size of the CPU virtual address range (in bytes). -+ * @offset: The offset from the start of the allocation to the specified CPU -+ * virtual address. -+ * -+ * Return: 0 if offset was obtained successfully. Error code otherwise. -+ */ -+int kbasep_find_enclosing_cpu_mapping_offset( -+ struct kbase_context *kctx, -+ unsigned long uaddr, size_t size, u64 *offset); -+ -+enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); -+void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); -+void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); -+ -+/** -+* @brief Allocates physical pages. -+* -+* Allocates \a nr_pages_requested and updates the alloc object. -+* -+* @param[in] alloc allocation object to add pages to -+* @param[in] nr_pages_requested number of physical pages to allocate -+* -+* @return 0 if all pages have been successfully allocated. Error code otherwise -+*/ -+int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested); -+ -+/** -+* @brief Free physical pages. -+* -+* Frees \a nr_pages and updates the alloc object. -+* -+* @param[in] alloc allocation object to free pages from -+* @param[in] nr_pages_to_free number of physical pages to free -+*/ -+int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); -+ -+static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) -+{ -+ SetPagePrivate(p); -+ if (sizeof(dma_addr_t) > sizeof(p->private)) { -+ /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the -+ * private field stays the same. So we have to be clever and -+ * use the fact that we only store DMA addresses of whole pages, -+ * so the low bits should be zero */ -+ KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); -+ set_page_private(p, dma_addr >> PAGE_SHIFT); -+ } else { -+ set_page_private(p, dma_addr); -+ } -+} -+ -+static inline dma_addr_t kbase_dma_addr(struct page *p) -+{ -+ if (sizeof(dma_addr_t) > sizeof(p->private)) -+ return ((dma_addr_t)page_private(p)) << PAGE_SHIFT; -+ -+ return (dma_addr_t)page_private(p); -+} -+ -+static inline void kbase_clear_dma_addr(struct page *p) -+{ -+ ClearPagePrivate(p); -+} -+ -+/** -+* @brief Process a bus or page fault. -+* -+* This function will process a fault on a specific address space -+* -+* @param[in] kbdev The @ref kbase_device the fault happened on -+* @param[in] kctx The @ref kbase_context for the faulting address space if -+* one was found. -+* @param[in] as The address space that has the fault -+*/ -+void kbase_mmu_interrupt_process(struct kbase_device *kbdev, -+ struct kbase_context *kctx, struct kbase_as *as); -+ -+/** -+ * @brief Process a page fault. -+ * -+ * @param[in] data work_struct passed by queue_work() -+ */ -+void page_fault_worker(struct work_struct *data); -+ -+/** -+ * @brief Process a bus fault. -+ * -+ * @param[in] data work_struct passed by queue_work() -+ */ -+void bus_fault_worker(struct work_struct *data); -+ -+/** -+ * @brief Flush MMU workqueues. -+ * -+ * This function will cause any outstanding page or bus faults to be processed. -+ * It should be called prior to powering off the GPU. -+ * -+ * @param[in] kbdev Device pointer -+ */ -+void kbase_flush_mmu_wqs(struct kbase_device *kbdev); -+ -+/** -+ * kbase_sync_single_for_device - update physical memory and give GPU ownership -+ * @kbdev: Device pointer -+ * @handle: DMA address of region -+ * @size: Size of region to sync -+ * @dir: DMA data direction -+ */ -+ -+void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, -+ size_t size, enum dma_data_direction dir); -+ -+/** -+ * kbase_sync_single_for_cpu - update physical memory and give CPU ownership -+ * @kbdev: Device pointer -+ * @handle: DMA address of region -+ * @size: Size of region to sync -+ * @dir: DMA data direction -+ */ -+ -+void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, -+ size_t size, enum dma_data_direction dir); -+ -+#ifdef CONFIG_DEBUG_FS -+/** -+ * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. -+ * @kctx: kbase context -+ */ -+void kbase_jit_debugfs_init(struct kbase_context *kctx); -+#endif /* CONFIG_DEBUG_FS */ -+ -+/** -+ * kbase_jit_init - Initialize the JIT memory pool management -+ * @kctx: kbase context -+ * -+ * Returns zero on success or negative error number on failure. -+ */ -+int kbase_jit_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_jit_allocate - Allocate JIT memory -+ * @kctx: kbase context -+ * @info: JIT allocation information -+ * -+ * Return: JIT allocation on success or NULL on failure. -+ */ -+struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, -+ struct base_jit_alloc_info *info); -+ -+/** -+ * kbase_jit_free - Free a JIT allocation -+ * @kctx: kbase context -+ * @reg: JIT allocation -+ * -+ * Frees a JIT allocation and places it into the free pool for later reuse. -+ */ -+void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg); -+ -+/** -+ * kbase_jit_backing_lost - Inform JIT that an allocation has lost backing -+ * @reg: JIT allocation -+ */ -+void kbase_jit_backing_lost(struct kbase_va_region *reg); -+ -+/** -+ * kbase_jit_evict - Evict a JIT allocation from the pool -+ * @kctx: kbase context -+ * -+ * Evict the least recently used JIT allocation from the pool. This can be -+ * required if normal VA allocations are failing due to VA exhaustion. -+ * -+ * Return: True if a JIT allocation was freed, false otherwise. -+ */ -+bool kbase_jit_evict(struct kbase_context *kctx); -+ -+/** -+ * kbase_jit_term - Terminate the JIT memory pool management -+ * @kctx: kbase context -+ */ -+void kbase_jit_term(struct kbase_context *kctx); -+ -+/** -+ * kbase_map_external_resource - Map an external resource to the GPU. -+ * @kctx: kbase context. -+ * @reg: The region to map. -+ * @locked_mm: The mm_struct which has been locked for this operation. -+ * @kds_res_count: The number of KDS resources. -+ * @kds_resources: Array of KDS resources. -+ * @kds_access_bitmap: Access bitmap for KDS. -+ * @exclusive: If the KDS resource requires exclusive access. -+ * -+ * Return: The physical allocation which backs the region on success or NULL -+ * on failure. -+ */ -+struct kbase_mem_phy_alloc *kbase_map_external_resource( -+ struct kbase_context *kctx, struct kbase_va_region *reg, -+ struct mm_struct *locked_mm -+#ifdef CONFIG_KDS -+ , u32 *kds_res_count, struct kds_resource **kds_resources, -+ unsigned long *kds_access_bitmap, bool exclusive -+#endif -+ ); -+ -+/** -+ * kbase_unmap_external_resource - Unmap an external resource from the GPU. -+ * @kctx: kbase context. -+ * @reg: The region to unmap or NULL if it has already been released. -+ * @alloc: The physical allocation being unmapped. -+ */ -+void kbase_unmap_external_resource(struct kbase_context *kctx, -+ struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc); -+ -+/** -+ * kbase_sticky_resource_init - Initialize sticky resource management. -+ * @kctx: kbase context -+ * -+ * Returns zero on success or negative error number on failure. -+ */ -+int kbase_sticky_resource_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_sticky_resource_acquire - Acquire a reference on a sticky resource. -+ * @kctx: kbase context. -+ * @gpu_addr: The GPU address of the external resource. -+ * -+ * Return: The metadata object which represents the binding between the -+ * external resource and the kbase context on success or NULL on failure. -+ */ -+struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( -+ struct kbase_context *kctx, u64 gpu_addr); -+ -+/** -+ * kbase_sticky_resource_release - Release a reference on a sticky resource. -+ * @kctx: kbase context. -+ * @meta: Binding metadata. -+ * @gpu_addr: GPU address of the external resource. -+ * -+ * If meta is NULL then gpu_addr will be used to scan the metadata list and -+ * find the matching metadata (if any), otherwise the provided meta will be -+ * used and gpu_addr will be ignored. -+ * -+ * Return: True if the release found the metadata and the reference was dropped. -+ */ -+bool kbase_sticky_resource_release(struct kbase_context *kctx, -+ struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); -+ -+/** -+ * kbase_sticky_resource_term - Terminate sticky resource management. -+ * @kctx: kbase context -+ */ -+void kbase_sticky_resource_term(struct kbase_context *kctx); -+ -+#endif /* _KBASE_MEM_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c -new file mode 100755 -index 000000000..e20315e67 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c -@@ -0,0 +1,2578 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_mem_linux.c -+ * Base kernel memory APIs, Linux implementation. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ -+ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+#include -+#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+#include -+#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+ -+static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); -+ -+/** -+ * kbase_mem_shrink_cpu_mapping - Shrink the CPU mapping(s) of an allocation -+ * @kctx: Context the region belongs to -+ * @reg: The GPU region -+ * @new_pages: The number of pages after the shrink -+ * @old_pages: The number of pages before the shrink -+ * -+ * Shrink (or completely remove) all CPU mappings which reference the shrunk -+ * part of the allocation. -+ * -+ * Note: Caller must be holding the processes mmap_lock lock. -+ */ -+static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, -+ struct kbase_va_region *reg, -+ u64 new_pages, u64 old_pages); -+ -+/** -+ * kbase_mem_shrink_gpu_mapping - Shrink the GPU mapping of an allocation -+ * @kctx: Context the region belongs to -+ * @reg: The GPU region or NULL if there isn't one -+ * @new_pages: The number of pages after the shrink -+ * @old_pages: The number of pages before the shrink -+ * -+ * Return: 0 on success, negative -errno on error -+ * -+ * Unmap the shrunk pages from the GPU mapping. Note that the size of the region -+ * itself is unmodified as we still need to reserve the VA, only the page tables -+ * will be modified by this function. -+ */ -+static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, -+ struct kbase_va_region *reg, -+ u64 new_pages, u64 old_pages); -+ -+struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, -+ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, -+ u64 *gpu_va) -+{ -+ int zone; -+ int gpu_pc_bits; -+ int cpu_va_bits; -+ struct kbase_va_region *reg; -+ struct device *dev; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(flags); -+ KBASE_DEBUG_ASSERT(gpu_va); -+ -+ dev = kctx->kbdev->dev; -+ *gpu_va = 0; /* return 0 on failure */ -+ -+ gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; -+ cpu_va_bits = BITS_PER_LONG; -+ -+ if (0 == va_pages) { -+ dev_warn(dev, "kbase_mem_alloc called with 0 va_pages!"); -+ goto bad_size; -+ } -+ -+ if (va_pages > (U64_MAX / PAGE_SIZE)) -+ /* 64-bit address range is the max */ -+ goto bad_size; -+ -+#if defined(CONFIG_64BIT) -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ cpu_va_bits = 32; -+#endif -+ -+ if (!kbase_check_alloc_flags(*flags)) { -+ dev_warn(dev, -+ "kbase_mem_alloc called with bad flags (%llx)", -+ (unsigned long long)*flags); -+ goto bad_flags; -+ } -+ -+ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && -+ !kbase_device_is_cpu_coherent(kctx->kbdev)) { -+ dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable"); -+ goto bad_flags; -+ } -+ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && -+ !kbase_device_is_cpu_coherent(kctx->kbdev)) { -+ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ -+ *flags &= ~BASE_MEM_COHERENT_SYSTEM; -+ } -+ -+ /* Limit GPU executable allocs to GPU PC size */ -+ if ((*flags & BASE_MEM_PROT_GPU_EX) && -+ (va_pages > (1ULL << gpu_pc_bits >> PAGE_SHIFT))) -+ goto bad_ex_size; -+ -+ /* find out which VA zone to use */ -+ if (*flags & BASE_MEM_SAME_VA) -+ zone = KBASE_REG_ZONE_SAME_VA; -+ else if (*flags & BASE_MEM_PROT_GPU_EX) -+ zone = KBASE_REG_ZONE_EXEC; -+ else -+ zone = KBASE_REG_ZONE_CUSTOM_VA; -+ -+ reg = kbase_alloc_free_region(kctx, 0, va_pages, zone); -+ if (!reg) { -+ dev_err(dev, "Failed to allocate free region"); -+ goto no_region; -+ } -+ -+ if (kbase_update_region_flags(kctx, reg, *flags) != 0) -+ goto invalid_flags; -+ -+ if (kbase_reg_prepare_native(reg, kctx) != 0) { -+ dev_err(dev, "Failed to prepare region"); -+ goto prepare_failed; -+ } -+ -+ if (*flags & BASE_MEM_GROW_ON_GPF) -+ reg->extent = extent; -+ else -+ reg->extent = 0; -+ -+ if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { -+ dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", -+ (unsigned long long)commit_pages, -+ (unsigned long long)va_pages); -+ goto no_mem; -+ } -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ /* mmap needed to setup VA? */ -+ if (*flags & BASE_MEM_SAME_VA) { -+ unsigned long prot = PROT_NONE; -+ unsigned long va_size = va_pages << PAGE_SHIFT; -+ unsigned long va_map = va_size; -+ unsigned long cookie, cookie_nr; -+ unsigned long cpu_addr; -+ -+ /* Bind to a cookie */ -+ if (!kctx->cookies) { -+ dev_err(dev, "No cookies available for allocation!"); -+ kbase_gpu_vm_unlock(kctx); -+ goto no_cookie; -+ } -+ /* return a cookie */ -+ cookie_nr = __ffs(kctx->cookies); -+ kctx->cookies &= ~(1UL << cookie_nr); -+ BUG_ON(kctx->pending_regions[cookie_nr]); -+ kctx->pending_regions[cookie_nr] = reg; -+ -+ kbase_gpu_vm_unlock(kctx); -+ -+ /* relocate to correct base */ -+ cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); -+ cookie <<= PAGE_SHIFT; -+ -+ /* -+ * 10.1-10.4 UKU userland relies on the kernel to call mmap. -+ * For all other versions we can just return the cookie -+ */ -+ if (kctx->api_version < KBASE_API_VERSION(10, 1) || -+ kctx->api_version > KBASE_API_VERSION(10, 4)) { -+ *gpu_va = (u64) cookie; -+ return reg; -+ } -+ if (*flags & BASE_MEM_PROT_CPU_RD) -+ prot |= PROT_READ; -+ if (*flags & BASE_MEM_PROT_CPU_WR) -+ prot |= PROT_WRITE; -+ -+ cpu_addr = vm_mmap(kctx->filp, 0, va_map, prot, -+ MAP_SHARED, cookie); -+ -+ if (IS_ERR_VALUE(cpu_addr)) { -+ kbase_gpu_vm_lock(kctx); -+ kctx->pending_regions[cookie_nr] = NULL; -+ kctx->cookies |= (1UL << cookie_nr); -+ kbase_gpu_vm_unlock(kctx); -+ goto no_mmap; -+ } -+ -+ *gpu_va = (u64) cpu_addr; -+ } else /* we control the VA */ { -+ if (kbase_gpu_mmap(kctx, reg, 0, va_pages, 1) != 0) { -+ dev_warn(dev, "Failed to map memory on GPU"); -+ kbase_gpu_vm_unlock(kctx); -+ goto no_mmap; -+ } -+ /* return real GPU VA */ -+ *gpu_va = reg->start_pfn << PAGE_SHIFT; -+ -+ kbase_gpu_vm_unlock(kctx); -+ } -+ -+ return reg; -+ -+no_mmap: -+no_cookie: -+no_mem: -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+invalid_flags: -+prepare_failed: -+ kfree(reg); -+no_region: -+bad_ex_size: -+bad_flags: -+bad_size: -+ return NULL; -+} -+KBASE_EXPORT_TEST_API(kbase_mem_alloc); -+ -+int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 * const out) -+{ -+ struct kbase_va_region *reg; -+ int ret = -EINVAL; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(out); -+ -+ if (gpu_addr & ~PAGE_MASK) { -+ dev_warn(kctx->kbdev->dev, "mem_query: gpu_addr: passed parameter is invalid"); -+ return -EINVAL; -+ } -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ /* Validate the region */ -+ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); -+ if (!reg || (reg->flags & KBASE_REG_FREE)) -+ goto out_unlock; -+ -+ switch (query) { -+ case KBASE_MEM_QUERY_COMMIT_SIZE: -+ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_ALIAS) { -+ *out = kbase_reg_current_backed_size(reg); -+ } else { -+ size_t i; -+ struct kbase_aliased *aliased; -+ *out = 0; -+ aliased = reg->cpu_alloc->imported.alias.aliased; -+ for (i = 0; i < reg->cpu_alloc->imported.alias.nents; i++) -+ *out += aliased[i].length; -+ } -+ break; -+ case KBASE_MEM_QUERY_VA_SIZE: -+ *out = reg->nr_pages; -+ break; -+ case KBASE_MEM_QUERY_FLAGS: -+ { -+ *out = 0; -+ if (KBASE_REG_CPU_WR & reg->flags) -+ *out |= BASE_MEM_PROT_CPU_WR; -+ if (KBASE_REG_CPU_RD & reg->flags) -+ *out |= BASE_MEM_PROT_CPU_RD; -+ if (KBASE_REG_CPU_CACHED & reg->flags) -+ *out |= BASE_MEM_CACHED_CPU; -+ if (KBASE_REG_GPU_WR & reg->flags) -+ *out |= BASE_MEM_PROT_GPU_WR; -+ if (KBASE_REG_GPU_RD & reg->flags) -+ *out |= BASE_MEM_PROT_GPU_RD; -+ if (!(KBASE_REG_GPU_NX & reg->flags)) -+ *out |= BASE_MEM_PROT_GPU_EX; -+ if (KBASE_REG_SHARE_BOTH & reg->flags) -+ *out |= BASE_MEM_COHERENT_SYSTEM; -+ if (KBASE_REG_SHARE_IN & reg->flags) -+ *out |= BASE_MEM_COHERENT_LOCAL; -+ break; -+ } -+ default: -+ *out = 0; -+ goto out_unlock; -+ } -+ -+ ret = 0; -+ -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return ret; -+} -+ -+/** -+ * kbase_mem_evictable_reclaim_count_objects - Count number of pages in the -+ * Ephemeral memory eviction list. -+ * @s: Shrinker -+ * @sc: Shrinker control -+ * -+ * Return: Number of pages which can be freed. -+ */ -+static -+unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, -+ struct shrink_control *sc) -+{ -+ struct kbase_context *kctx; -+ struct kbase_mem_phy_alloc *alloc; -+ unsigned long pages = 0; -+ -+ kctx = container_of(s, struct kbase_context, reclaim); -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ -+ list_for_each_entry(alloc, &kctx->evict_list, evict_node) -+ pages += alloc->nents; -+ -+ mutex_unlock(&kctx->jit_evict_lock); -+ return pages; -+} -+ -+/** -+ * kbase_mem_evictable_reclaim_scan_objects - Scan the Ephemeral memory eviction -+ * list for pages and try to reclaim them. -+ * @s: Shrinker -+ * @sc: Shrinker control -+ * -+ * Return: Number of pages freed (can be less then requested) or -1 if the -+ * shrinker failed to free pages in its pool. -+ * -+ * Note: -+ * This function accesses region structures without taking the region lock, -+ * this is required as the OOM killer can call the shrinker after the region -+ * lock has already been held. -+ * This is safe as we can guarantee that a region on the eviction list will -+ * not be freed (kbase_mem_free_region removes the allocation from the list -+ * before destroying it), or modified by other parts of the driver. -+ * The eviction list itself is guarded by the eviction lock and the MMU updates -+ * are protected by their own lock. -+ */ -+static -+unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, -+ struct shrink_control *sc) -+{ -+ struct kbase_context *kctx; -+ struct kbase_mem_phy_alloc *alloc; -+ struct kbase_mem_phy_alloc *tmp; -+ unsigned long freed = 0; -+ -+ kctx = container_of(s, struct kbase_context, reclaim); -+ mutex_lock(&kctx->jit_evict_lock); -+ -+ list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { -+ int err; -+ -+ err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, -+ 0, alloc->nents); -+ if (err != 0) { -+ /* -+ * Failed to remove GPU mapping, tell the shrinker -+ * to stop trying to shrink our slab even though we -+ * have pages in it. -+ */ -+ freed = -1; -+ goto out_unlock; -+ } -+ -+ /* -+ * Update alloc->evicted before freeing the backing so the -+ * helper can determine that it needs to bypass the accounting -+ * and memory pool. -+ */ -+ alloc->evicted = alloc->nents; -+ -+ kbase_free_phy_pages_helper(alloc, alloc->evicted); -+ freed += alloc->evicted; -+ list_del_init(&alloc->evict_node); -+ -+ /* -+ * Inform the JIT allocator this region has lost backing -+ * as it might need to free the allocation. -+ */ -+ kbase_jit_backing_lost(alloc->reg); -+ -+ /* Enough pages have been freed so stop now */ -+ if (freed > sc->nr_to_scan) -+ break; -+ } -+out_unlock: -+ mutex_unlock(&kctx->jit_evict_lock); -+ -+ return freed; -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, -+ struct shrink_control *sc) -+{ -+ if (sc->nr_to_scan == 0) -+ return kbase_mem_evictable_reclaim_count_objects(s, sc); -+ -+ return kbase_mem_evictable_reclaim_scan_objects(s, sc); -+} -+#endif -+ -+int kbase_mem_evictable_init(struct kbase_context *kctx) -+{ -+ INIT_LIST_HEAD(&kctx->evict_list); -+ mutex_init(&kctx->jit_evict_lock); -+ -+ /* Register shrinker */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+ kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; -+#else -+ kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; -+ kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; -+#endif -+ kctx->reclaim.seeks = DEFAULT_SEEKS; -+ /* Kernel versions prior to 3.1 : -+ * struct shrinker does not define batch */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) -+ kctx->reclaim.batch = 0; -+#endif -+ register_shrinker(&kctx->reclaim); -+ return 0; -+} -+ -+void kbase_mem_evictable_deinit(struct kbase_context *kctx) -+{ -+ unregister_shrinker(&kctx->reclaim); -+} -+ -+/** -+ * kbase_mem_evictable_mark_reclaim - Mark the pages as reclaimable. -+ * @alloc: The physical allocation -+ */ -+static void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) -+{ -+ struct kbase_context *kctx = alloc->imported.kctx; -+ int __maybe_unused new_page_count; -+ -+ kbase_process_page_usage_dec(kctx, alloc->nents); -+ new_page_count = kbase_atomic_sub_pages(alloc->nents, -+ &kctx->used_pages); -+ kbase_atomic_sub_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); -+ -+ KBASE_TLSTREAM_AUX_PAGESALLOC( -+ (u32)kctx->id, -+ (u64)new_page_count); -+} -+ -+/** -+ * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. -+ * @alloc: The physical allocation -+ */ -+static -+void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) -+{ -+ struct kbase_context *kctx = alloc->imported.kctx; -+ int __maybe_unused new_page_count; -+ -+ new_page_count = kbase_atomic_add_pages(alloc->nents, -+ &kctx->used_pages); -+ kbase_atomic_add_pages(alloc->nents, &kctx->kbdev->memdev.used_pages); -+ -+ /* Increase mm counters so that the allocation is accounted for -+ * against the process and thus is visible to the OOM killer. -+ */ -+ kbase_process_page_usage_inc(kctx, alloc->nents); -+ -+ KBASE_TLSTREAM_AUX_PAGESALLOC( -+ (u32)kctx->id, -+ (u64)new_page_count); -+} -+ -+int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) -+{ -+ struct kbase_context *kctx = gpu_alloc->imported.kctx; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* This alloction can't already be on a list. */ -+ WARN_ON(!list_empty(&gpu_alloc->evict_node)); -+ -+ kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, -+ 0, gpu_alloc->nents); -+ -+ /* -+ * Add the allocation to the eviction list, after this point the shrink -+ * can reclaim it. -+ */ -+ mutex_lock(&kctx->jit_evict_lock); -+ list_add(&gpu_alloc->evict_node, &kctx->evict_list); -+ mutex_unlock(&kctx->jit_evict_lock); -+ kbase_mem_evictable_mark_reclaim(gpu_alloc); -+ -+ gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; -+ return 0; -+} -+ -+bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) -+{ -+ struct kbase_context *kctx = gpu_alloc->imported.kctx; -+ int err = 0; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* -+ * First remove the allocation from the eviction list as it's no -+ * longer eligible for eviction. -+ */ -+ list_del_init(&gpu_alloc->evict_node); -+ -+ if (gpu_alloc->evicted == 0) { -+ /* -+ * The backing is still present, update the VM stats as it's -+ * in use again. -+ */ -+ kbase_mem_evictable_unmark_reclaim(gpu_alloc); -+ } else { -+ /* If the region is still alive ... */ -+ if (gpu_alloc->reg) { -+ /* ... allocate replacement backing ... */ -+ err = kbase_alloc_phy_pages_helper(gpu_alloc, -+ gpu_alloc->evicted); -+ -+ /* -+ * ... and grow the mapping back to its -+ * pre-eviction size. -+ */ -+ if (!err) -+ err = kbase_mem_grow_gpu_mapping(kctx, -+ gpu_alloc->reg, -+ gpu_alloc->evicted, 0); -+ -+ gpu_alloc->evicted = 0; -+ } -+ } -+ -+ /* If the region is still alive remove the DONT_NEED attribute. */ -+ if (gpu_alloc->reg) -+ gpu_alloc->reg->flags &= ~KBASE_REG_DONT_NEED; -+ -+ return (err == 0); -+} -+ -+int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) -+{ -+ struct kbase_va_region *reg; -+ int ret = -EINVAL; -+ unsigned int real_flags = 0; -+ unsigned int prev_flags = 0; -+ bool prev_needed, new_needed; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ if (!gpu_addr) -+ return -EINVAL; -+ -+ if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) -+ return -EINVAL; -+ -+ /* nuke other bits */ -+ flags &= mask; -+ -+ /* check for only supported flags */ -+ if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) -+ goto out; -+ -+ /* mask covers bits we don't support? */ -+ if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) -+ goto out; -+ -+ /* convert flags */ -+ if (BASE_MEM_COHERENT_SYSTEM & flags) -+ real_flags |= KBASE_REG_SHARE_BOTH; -+ else if (BASE_MEM_COHERENT_LOCAL & flags) -+ real_flags |= KBASE_REG_SHARE_IN; -+ -+ /* now we can lock down the context, and find the region */ -+ down_write(¤t->mm->mmap_lock); -+ kbase_gpu_vm_lock(kctx); -+ -+ /* Validate the region */ -+ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); -+ if (!reg || (reg->flags & KBASE_REG_FREE)) -+ goto out_unlock; -+ -+ /* Is the region being transitioning between not needed and needed? */ -+ prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; -+ new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; -+ if (prev_needed != new_needed) { -+ /* Aliased allocations can't be made ephemeral */ -+ if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) -+ goto out_unlock; -+ -+ if (new_needed) { -+ /* Only native allocations can be marked not needed */ -+ if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ ret = kbase_mem_evictable_make(reg->gpu_alloc); -+ if (ret) -+ goto out_unlock; -+ } else { -+ kbase_mem_evictable_unmake(reg->gpu_alloc); -+ } -+ } -+ -+ /* limit to imported memory */ -+ if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) && -+ (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) -+ goto out_unlock; -+ -+ /* no change? */ -+ if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { -+ ret = 0; -+ goto out_unlock; -+ } -+ -+ /* save for roll back */ -+ prev_flags = reg->flags; -+ reg->flags &= ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); -+ reg->flags |= real_flags; -+ -+ /* Currently supporting only imported memory */ -+ switch (reg->gpu_alloc->type) { -+#ifdef CONFIG_UMP -+ case KBASE_MEM_TYPE_IMPORTED_UMP: -+ ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_cpu_phy_pages(reg), reg->gpu_alloc->nents, reg->flags); -+ break; -+#endif -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ case KBASE_MEM_TYPE_IMPORTED_UMM: -+ /* Future use will use the new flags, existing mapping will NOT be updated -+ * as memory should not be in use by the GPU when updating the flags. -+ */ -+ ret = 0; -+ WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); -+ break; -+#endif -+ default: -+ break; -+ } -+ -+ /* roll back on error, i.e. not UMP */ -+ if (ret) -+ reg->flags = prev_flags; -+ -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ up_write(¤t->mm->mmap_lock); -+out: -+ return ret; -+} -+ -+#define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) -+ -+#ifdef CONFIG_UMP -+static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags) -+{ -+ struct kbase_va_region *reg; -+ ump_dd_handle umph; -+ u64 block_count; -+ const ump_dd_physical_block_64 *block_array; -+ u64 i, j; -+ int page = 0; -+ ump_alloc_flags ump_flags; -+ ump_alloc_flags cpu_flags; -+ ump_alloc_flags gpu_flags; -+ -+ if (*flags & BASE_MEM_SECURE) -+ goto bad_flags; -+ -+ umph = ump_dd_from_secure_id(id); -+ if (UMP_DD_INVALID_MEMORY_HANDLE == umph) -+ goto bad_id; -+ -+ ump_flags = ump_dd_allocation_flags_get(umph); -+ cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK; -+ gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) & -+ UMP_DEVICE_MASK; -+ -+ *va_pages = ump_dd_size_get_64(umph); -+ *va_pages >>= PAGE_SHIFT; -+ -+ if (!*va_pages) -+ goto bad_size; -+ -+ if (*va_pages > (U64_MAX / PAGE_SIZE)) -+ /* 64-bit address range is the max */ -+ goto bad_size; -+ -+ if (*flags & BASE_MEM_SAME_VA) -+ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); -+ else -+ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); -+ -+ if (!reg) -+ goto no_region; -+ -+ /* we've got pages to map now, and support SAME_VA */ -+ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; -+ -+ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP); -+ if (IS_ERR_OR_NULL(reg->gpu_alloc)) -+ goto no_alloc_obj; -+ -+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ -+ reg->gpu_alloc->imported.ump_handle = umph; -+ -+ reg->flags &= ~KBASE_REG_FREE; -+ reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */ -+ reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */ -+ -+ /* Override import flags based on UMP flags */ -+ *flags &= ~(BASE_MEM_CACHED_CPU); -+ *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR); -+ *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR); -+ -+ if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == -+ (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) { -+ reg->flags |= KBASE_REG_CPU_CACHED; -+ *flags |= BASE_MEM_CACHED_CPU; -+ } -+ -+ if (cpu_flags & UMP_PROT_CPU_WR) { -+ reg->flags |= KBASE_REG_CPU_WR; -+ *flags |= BASE_MEM_PROT_CPU_WR; -+ } -+ -+ if (cpu_flags & UMP_PROT_CPU_RD) { -+ reg->flags |= KBASE_REG_CPU_RD; -+ *flags |= BASE_MEM_PROT_CPU_RD; -+ } -+ -+ if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == -+ (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) -+ reg->flags |= KBASE_REG_GPU_CACHED; -+ -+ if (gpu_flags & UMP_PROT_DEVICE_WR) { -+ reg->flags |= KBASE_REG_GPU_WR; -+ *flags |= BASE_MEM_PROT_GPU_WR; -+ } -+ -+ if (gpu_flags & UMP_PROT_DEVICE_RD) { -+ reg->flags |= KBASE_REG_GPU_RD; -+ *flags |= BASE_MEM_PROT_GPU_RD; -+ } -+ -+ /* ump phys block query */ -+ ump_dd_phys_blocks_get_64(umph, &block_count, &block_array); -+ -+ for (i = 0; i < block_count; i++) { -+ for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) { -+ reg->gpu_alloc->pages[page] = block_array[i].addr + (j << PAGE_SHIFT); -+ page++; -+ } -+ } -+ reg->gpu_alloc->nents = *va_pages; -+ reg->extent = 0; -+ -+ return reg; -+ -+no_alloc_obj: -+ kfree(reg); -+no_region: -+bad_size: -+ ump_dd_release(umph); -+bad_id: -+bad_flags: -+ return NULL; -+} -+#endif /* CONFIG_UMP */ -+ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, -+ int fd, u64 *va_pages, u64 *flags, u32 padding) -+{ -+ struct kbase_va_region *reg; -+ struct dma_buf *dma_buf; -+ struct dma_buf_attachment *dma_attachment; -+ bool shared_zone = false; -+ -+ dma_buf = dma_buf_get(fd); -+ if (IS_ERR_OR_NULL(dma_buf)) -+ goto no_buf; -+ -+ dma_attachment = dma_buf_attach(dma_buf, kctx->kbdev->dev); -+ if (!dma_attachment) -+ goto no_attachment; -+ -+ *va_pages = (PAGE_ALIGN(dma_buf->size) >> PAGE_SHIFT) + padding; -+ if (!*va_pages) -+ goto bad_size; -+ -+ if (*va_pages > (U64_MAX / PAGE_SIZE)) -+ /* 64-bit address range is the max */ -+ goto bad_size; -+ -+ /* ignore SAME_VA */ -+ *flags &= ~BASE_MEM_SAME_VA; -+ -+ if (*flags & BASE_MEM_IMPORT_SHARED) -+ shared_zone = true; -+ -+#ifdef CONFIG_64BIT -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ /* -+ * 64-bit tasks require us to reserve VA on the CPU that we use -+ * on the GPU. -+ */ -+ shared_zone = true; -+ } -+#endif -+ -+ if (shared_zone) { -+ *flags |= BASE_MEM_NEED_MMAP; -+ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); -+ } else { -+ reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); -+ } -+ -+ if (!reg) -+ goto no_region; -+ -+ reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMM); -+ if (IS_ERR_OR_NULL(reg->gpu_alloc)) -+ goto no_alloc_obj; -+ -+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ -+ /* No pages to map yet */ -+ reg->gpu_alloc->nents = 0; -+ -+ if (kbase_update_region_flags(kctx, reg, *flags) != 0) -+ goto invalid_flags; -+ -+ reg->flags &= ~KBASE_REG_FREE; -+ reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ -+ reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ -+ reg->flags |= KBASE_REG_GPU_CACHED; -+ -+ if (*flags & BASE_MEM_SECURE) -+ reg->flags |= KBASE_REG_SECURE; -+ -+ if (padding) -+ reg->flags |= KBASE_REG_IMPORT_PAD; -+ -+ reg->gpu_alloc->type = KBASE_MEM_TYPE_IMPORTED_UMM; -+ reg->gpu_alloc->imported.umm.sgt = NULL; -+ reg->gpu_alloc->imported.umm.dma_buf = dma_buf; -+ reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; -+ reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; -+ reg->extent = 0; -+ -+ return reg; -+ -+invalid_flags: -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+no_alloc_obj: -+ kfree(reg); -+no_region: -+bad_size: -+ dma_buf_detach(dma_buf, dma_attachment); -+no_attachment: -+ dma_buf_put(dma_buf); -+no_buf: -+ return NULL; -+} -+#endif /* CONFIG_DMA_SHARED_BUFFER */ -+ -+static u32 kbase_get_cache_line_alignment(struct kbase_context *kctx) -+{ -+ u32 cpu_cache_line_size = cache_line_size(); -+ u32 gpu_cache_line_size = -+ (1UL << kctx->kbdev->gpu_props.props.l2_props.log2_line_size); -+ -+ return ((cpu_cache_line_size > gpu_cache_line_size) ? -+ cpu_cache_line_size : -+ gpu_cache_line_size); -+} -+ -+static struct kbase_va_region *kbase_mem_from_user_buffer( -+ struct kbase_context *kctx, unsigned long address, -+ unsigned long size, u64 *va_pages, u64 *flags) -+{ -+ long i; -+ struct kbase_va_region *reg; -+ long faulted_pages; -+ int zone = KBASE_REG_ZONE_CUSTOM_VA; -+ bool shared_zone = false; -+ u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx); -+ struct kbase_alloc_import_user_buf *user_buf; -+ struct page **pages = NULL; -+ -+ if ((address & (cache_line_alignment - 1)) != 0 || -+ (size & (cache_line_alignment - 1)) != 0) { -+ /* Coherency must be enabled to handle partial cache lines */ -+ if (*flags & (BASE_MEM_COHERENT_SYSTEM | -+ BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { -+ /* Force coherent system required flag, import will -+ * then fail if coherency isn't available -+ */ -+ *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; -+ } else { -+ dev_warn(kctx->kbdev->dev, -+ "User buffer is not cache line aligned and no coherency enabled\n"); -+ goto bad_size; -+ } -+ } -+ -+ *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - -+ PFN_DOWN(address); -+ if (!*va_pages) -+ goto bad_size; -+ -+ if (*va_pages > (UINT64_MAX / PAGE_SIZE)) -+ /* 64-bit address range is the max */ -+ goto bad_size; -+ -+ /* SAME_VA generally not supported with imported memory (no known use cases) */ -+ *flags &= ~BASE_MEM_SAME_VA; -+ -+ if (*flags & BASE_MEM_IMPORT_SHARED) -+ shared_zone = true; -+ -+#ifdef CONFIG_64BIT -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ /* -+ * 64-bit tasks require us to reserve VA on the CPU that we use -+ * on the GPU. -+ */ -+ shared_zone = true; -+ } -+#endif -+ -+ if (shared_zone) { -+ *flags |= BASE_MEM_NEED_MMAP; -+ zone = KBASE_REG_ZONE_SAME_VA; -+ } -+ -+ reg = kbase_alloc_free_region(kctx, 0, *va_pages, zone); -+ -+ if (!reg) -+ goto no_region; -+ -+ reg->gpu_alloc = kbase_alloc_create(*va_pages, -+ KBASE_MEM_TYPE_IMPORTED_USER_BUF); -+ if (IS_ERR_OR_NULL(reg->gpu_alloc)) -+ goto no_alloc_obj; -+ -+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ -+ if (kbase_update_region_flags(kctx, reg, *flags) != 0) -+ goto invalid_flags; -+ -+ reg->flags &= ~KBASE_REG_FREE; -+ reg->flags |= KBASE_REG_GPU_NX; /* User-buffers are always No eXecute */ -+ reg->flags &= ~KBASE_REG_GROWABLE; /* Cannot be grown */ -+ reg->flags &= ~KBASE_REG_CPU_CACHED; -+ -+ user_buf = ®->gpu_alloc->imported.user_buf; -+ -+ user_buf->size = size; -+ user_buf->address = address; -+ user_buf->nr_pages = *va_pages; -+ user_buf->mm = current->mm; -+ user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), -+ GFP_KERNEL); -+ -+ if (!user_buf->pages) -+ goto no_page_array; -+ -+ /* If the region is coherent with the CPU then the memory is imported -+ * and mapped onto the GPU immediately. -+ * Otherwise get_user_pages is called as a sanity check, but with -+ * NULL as the pages argument which will fault the pages, but not -+ * pin them. The memory will then be pinned only around the jobs that -+ * specify the region as an external resource. -+ */ -+ if (reg->flags & KBASE_REG_SHARE_BOTH) { -+ pages = user_buf->pages; -+ *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; -+ } -+ -+ down_read(¤t->mm->mmap_lock); -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+ faulted_pages = get_user_pages(current, current->mm, address, *va_pages, -+ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); -+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+ faulted_pages = get_user_pages(address, *va_pages, -+ reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); -+#else -+ faulted_pages = get_user_pages(address, *va_pages, -+ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -+ pages, NULL); -+#endif -+ -+ up_read(¤t->mm->mmap_lock); -+ -+ if (faulted_pages != *va_pages) -+ goto fault_mismatch; -+ -+ atomic_inc(¤t->mm->mm_count); -+ -+ reg->gpu_alloc->nents = 0; -+ reg->extent = 0; -+ -+ if (pages) { -+ struct device *dev = kctx->kbdev->dev; -+ unsigned long local_size = user_buf->size; -+ unsigned long offset = user_buf->address & ~PAGE_MASK; -+ phys_addr_t *pa = kbase_get_gpu_phy_pages(reg); -+ -+ /* Top bit signifies that this was pinned on import */ -+ user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; -+ -+ for (i = 0; i < faulted_pages; i++) { -+ dma_addr_t dma_addr; -+ unsigned long min; -+ -+ min = MIN(PAGE_SIZE - offset, local_size); -+ dma_addr = dma_map_page(dev, pages[i], -+ offset, min, -+ DMA_BIDIRECTIONAL); -+ if (dma_mapping_error(dev, dma_addr)) -+ goto unwind_dma_map; -+ -+ user_buf->dma_addrs[i] = dma_addr; -+ pa[i] = page_to_phys(pages[i]); -+ -+ local_size -= min; -+ offset = 0; -+ } -+ -+ reg->gpu_alloc->nents = faulted_pages; -+ } -+ -+ return reg; -+ -+unwind_dma_map: -+ while (i--) { -+ dma_unmap_page(kctx->kbdev->dev, -+ user_buf->dma_addrs[i], -+ PAGE_SIZE, DMA_BIDIRECTIONAL); -+ } -+fault_mismatch: -+ if (pages) { -+ for (i = 0; i < faulted_pages; i++) -+ put_page(pages[i]); -+ } -+ kfree(user_buf->pages); -+no_page_array: -+invalid_flags: -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+no_alloc_obj: -+ kfree(reg); -+no_region: -+bad_size: -+ return NULL; -+ -+} -+ -+ -+u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, -+ u64 nents, struct base_mem_aliasing_info *ai, -+ u64 *num_pages) -+{ -+ struct kbase_va_region *reg; -+ u64 gpu_va; -+ size_t i; -+ bool coherent; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(flags); -+ KBASE_DEBUG_ASSERT(ai); -+ KBASE_DEBUG_ASSERT(num_pages); -+ -+ /* mask to only allowed flags */ -+ *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | -+ BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | -+ BASE_MEM_COHERENT_SYSTEM_REQUIRED); -+ -+ if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { -+ dev_warn(kctx->kbdev->dev, -+ "kbase_mem_alias called with bad flags (%llx)", -+ (unsigned long long)*flags); -+ goto bad_flags; -+ } -+ coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || -+ (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; -+ -+ if (!stride) -+ goto bad_stride; -+ -+ if (!nents) -+ goto bad_nents; -+ -+ if ((nents * stride) > (U64_MAX / PAGE_SIZE)) -+ /* 64-bit address range is the max */ -+ goto bad_size; -+ -+ /* calculate the number of pages this alias will cover */ -+ *num_pages = nents * stride; -+ -+#ifdef CONFIG_64BIT -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ /* 64-bit tasks must MMAP anyway, but not expose this address to -+ * clients */ -+ *flags |= BASE_MEM_NEED_MMAP; -+ reg = kbase_alloc_free_region(kctx, 0, *num_pages, -+ KBASE_REG_ZONE_SAME_VA); -+ } else { -+#else -+ if (1) { -+#endif -+ reg = kbase_alloc_free_region(kctx, 0, *num_pages, -+ KBASE_REG_ZONE_CUSTOM_VA); -+ } -+ -+ if (!reg) -+ goto no_reg; -+ -+ /* zero-sized page array, as we don't need one/can support one */ -+ reg->gpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_ALIAS); -+ if (IS_ERR_OR_NULL(reg->gpu_alloc)) -+ goto no_alloc_obj; -+ -+ reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ -+ if (kbase_update_region_flags(kctx, reg, *flags) != 0) -+ goto invalid_flags; -+ -+ reg->gpu_alloc->imported.alias.nents = nents; -+ reg->gpu_alloc->imported.alias.stride = stride; -+ reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); -+ if (!reg->gpu_alloc->imported.alias.aliased) -+ goto no_aliased_array; -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ /* validate and add src handles */ -+ for (i = 0; i < nents; i++) { -+ if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { -+ if (ai[i].handle.basep.handle != -+ BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) -+ goto bad_handle; /* unsupported magic handle */ -+ if (!ai[i].length) -+ goto bad_handle; /* must be > 0 */ -+ if (ai[i].length > stride) -+ goto bad_handle; /* can't be larger than the -+ stride */ -+ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; -+ } else { -+ struct kbase_va_region *aliasing_reg; -+ struct kbase_mem_phy_alloc *alloc; -+ -+ aliasing_reg = kbase_region_tracker_find_region_base_address( -+ kctx, -+ (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); -+ -+ /* validate found region */ -+ if (!aliasing_reg) -+ goto bad_handle; /* Not found */ -+ if (aliasing_reg->flags & KBASE_REG_FREE) -+ goto bad_handle; /* Free region */ -+ if (aliasing_reg->flags & KBASE_REG_DONT_NEED) -+ goto bad_handle; /* Ephemeral region */ -+ if (!aliasing_reg->gpu_alloc) -+ goto bad_handle; /* No alloc */ -+ if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) -+ goto bad_handle; /* Not a native alloc */ -+ if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) -+ goto bad_handle; -+ /* Non-coherent memory cannot alias -+ coherent memory, and vice versa.*/ -+ -+ /* check size against stride */ -+ if (!ai[i].length) -+ goto bad_handle; /* must be > 0 */ -+ if (ai[i].length > stride) -+ goto bad_handle; /* can't be larger than the -+ stride */ -+ -+ alloc = aliasing_reg->gpu_alloc; -+ -+ /* check against the alloc's size */ -+ if (ai[i].offset > alloc->nents) -+ goto bad_handle; /* beyond end */ -+ if (ai[i].offset + ai[i].length > alloc->nents) -+ goto bad_handle; /* beyond end */ -+ -+ reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); -+ reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; -+ reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; -+ } -+ } -+ -+#ifdef CONFIG_64BIT -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+ /* Bind to a cookie */ -+ if (!kctx->cookies) { -+ dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); -+ goto no_cookie; -+ } -+ /* return a cookie */ -+ gpu_va = __ffs(kctx->cookies); -+ kctx->cookies &= ~(1UL << gpu_va); -+ BUG_ON(kctx->pending_regions[gpu_va]); -+ kctx->pending_regions[gpu_va] = reg; -+ -+ /* relocate to correct base */ -+ gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); -+ gpu_va <<= PAGE_SHIFT; -+ } else /* we control the VA */ { -+#else -+ if (1) { -+#endif -+ if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1) != 0) { -+ dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); -+ goto no_mmap; -+ } -+ /* return real GPU VA */ -+ gpu_va = reg->start_pfn << PAGE_SHIFT; -+ } -+ -+ reg->flags &= ~KBASE_REG_FREE; -+ reg->flags &= ~KBASE_REG_GROWABLE; -+ -+ kbase_gpu_vm_unlock(kctx); -+ -+ return gpu_va; -+ -+#ifdef CONFIG_64BIT -+no_cookie: -+#endif -+no_mmap: -+bad_handle: -+ kbase_gpu_vm_unlock(kctx); -+no_aliased_array: -+invalid_flags: -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+no_alloc_obj: -+ kfree(reg); -+no_reg: -+bad_size: -+bad_nents: -+bad_stride: -+bad_flags: -+ return 0; -+} -+ -+int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, -+ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, -+ u64 *flags) -+{ -+ struct kbase_va_region *reg; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(gpu_va); -+ KBASE_DEBUG_ASSERT(va_pages); -+ KBASE_DEBUG_ASSERT(flags); -+ -+#ifdef CONFIG_64BIT -+ if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ *flags |= BASE_MEM_SAME_VA; -+#endif -+ -+ if (!kbase_check_import_flags(*flags)) { -+ dev_warn(kctx->kbdev->dev, -+ "kbase_mem_import called with bad flags (%llx)", -+ (unsigned long long)*flags); -+ goto bad_flags; -+ } -+ -+ if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && -+ !kbase_device_is_cpu_coherent(kctx->kbdev)) { -+ dev_warn(kctx->kbdev->dev, -+ "kbase_mem_import call required coherent mem when unavailable"); -+ goto bad_flags; -+ } -+ if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && -+ !kbase_device_is_cpu_coherent(kctx->kbdev)) { -+ /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ -+ *flags &= ~BASE_MEM_COHERENT_SYSTEM; -+ } -+ -+ if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { -+ dev_warn(kctx->kbdev->dev, -+ "padding is only supported for UMM"); -+ goto bad_flags; -+ } -+ -+ switch (type) { -+#ifdef CONFIG_UMP -+ case BASE_MEM_IMPORT_TYPE_UMP: { -+ ump_secure_id id; -+ -+ if (get_user(id, (ump_secure_id __user *)phandle)) -+ reg = NULL; -+ else -+ reg = kbase_mem_from_ump(kctx, id, va_pages, flags); -+ } -+ break; -+#endif /* CONFIG_UMP */ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ case BASE_MEM_IMPORT_TYPE_UMM: { -+ int fd; -+ -+ if (get_user(fd, (int __user *)phandle)) -+ reg = NULL; -+ else -+ reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, -+ padding); -+ } -+ break; -+#endif /* CONFIG_DMA_SHARED_BUFFER */ -+ case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { -+ struct base_mem_import_user_buffer user_buffer; -+ void __user *uptr; -+ -+ if (copy_from_user(&user_buffer, phandle, -+ sizeof(user_buffer))) { -+ reg = NULL; -+ } else { -+#ifdef CONFIG_COMPAT -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+ uptr = compat_ptr(user_buffer.ptr.compat_value); -+ else -+#endif -+ uptr = user_buffer.ptr.value; -+ -+ reg = kbase_mem_from_user_buffer(kctx, -+ (unsigned long)uptr, user_buffer.length, -+ va_pages, flags); -+ } -+ break; -+ } -+ default: { -+ reg = NULL; -+ break; -+ } -+ } -+ -+ if (!reg) -+ goto no_reg; -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ /* mmap needed to setup VA? */ -+ if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { -+ /* Bind to a cookie */ -+ if (!kctx->cookies) -+ goto no_cookie; -+ /* return a cookie */ -+ *gpu_va = __ffs(kctx->cookies); -+ kctx->cookies &= ~(1UL << *gpu_va); -+ BUG_ON(kctx->pending_regions[*gpu_va]); -+ kctx->pending_regions[*gpu_va] = reg; -+ -+ /* relocate to correct base */ -+ *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); -+ *gpu_va <<= PAGE_SHIFT; -+ -+ } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { -+ /* we control the VA, mmap now to the GPU */ -+ if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1) != 0) -+ goto no_gpu_va; -+ /* return real GPU VA */ -+ *gpu_va = reg->start_pfn << PAGE_SHIFT; -+ } else { -+ /* we control the VA, but nothing to mmap yet */ -+ if (kbase_add_va_region(kctx, reg, 0, *va_pages, 1) != 0) -+ goto no_gpu_va; -+ /* return real GPU VA */ -+ *gpu_va = reg->start_pfn << PAGE_SHIFT; -+ } -+ -+ /* clear out private flags */ -+ *flags &= ((1UL << BASE_MEM_FLAGS_NR_BITS) - 1); -+ -+ kbase_gpu_vm_unlock(kctx); -+ -+ return 0; -+ -+no_gpu_va: -+no_cookie: -+ kbase_gpu_vm_unlock(kctx); -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+ kfree(reg); -+no_reg: -+bad_flags: -+ *gpu_va = 0; -+ *va_pages = 0; -+ *flags = 0; -+ return -ENOMEM; -+} -+ -+int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, -+ struct kbase_va_region *reg, -+ u64 new_pages, u64 old_pages) -+{ -+ phys_addr_t *phy_pages; -+ u64 delta = new_pages - old_pages; -+ int ret = 0; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* Map the new pages into the GPU */ -+ phy_pages = kbase_get_gpu_phy_pages(reg); -+ ret = kbase_mmu_insert_pages(kctx, reg->start_pfn + old_pages, -+ phy_pages + old_pages, delta, reg->flags); -+ -+ return ret; -+} -+ -+static void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, -+ struct kbase_va_region *reg, -+ u64 new_pages, u64 old_pages) -+{ -+ u64 gpu_va_start = reg->start_pfn; -+ -+ if (new_pages == old_pages) -+ /* Nothing to do */ -+ return; -+ -+ unmap_mapping_range(kctx->filp->f_inode->i_mapping, -+ (gpu_va_start + new_pages)<start_pfn + new_pages, delta); -+ -+ return ret; -+} -+ -+int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) -+{ -+ u64 old_pages; -+ u64 delta; -+ int res = -EINVAL; -+ struct kbase_va_region *reg; -+ bool read_locked = false; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(gpu_addr != 0); -+ -+ if (gpu_addr & ~PAGE_MASK) { -+ dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); -+ return -EINVAL; -+ } -+ -+ down_write(¤t->mm->mmap_lock); -+ kbase_gpu_vm_lock(kctx); -+ -+ /* Validate the region */ -+ reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); -+ if (!reg || (reg->flags & KBASE_REG_FREE)) -+ goto out_unlock; -+ -+ KBASE_DEBUG_ASSERT(reg->cpu_alloc); -+ KBASE_DEBUG_ASSERT(reg->gpu_alloc); -+ -+ if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) -+ goto out_unlock; -+ -+ if (0 == (reg->flags & KBASE_REG_GROWABLE)) -+ goto out_unlock; -+ -+ /* Would overflow the VA region */ -+ if (new_pages > reg->nr_pages) -+ goto out_unlock; -+ -+ /* can't be mapped more than once on the GPU */ -+ if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) -+ goto out_unlock; -+ /* can't grow regions which are ephemeral */ -+ if (reg->flags & KBASE_REG_DONT_NEED) -+ goto out_unlock; -+ -+ if (new_pages == reg->gpu_alloc->nents) { -+ /* no change */ -+ res = 0; -+ goto out_unlock; -+ } -+ -+ old_pages = kbase_reg_current_backed_size(reg); -+ if (new_pages > old_pages) { -+ delta = new_pages - old_pages; -+ -+ /* -+ * No update to the mm so downgrade the writer lock to a read -+ * lock so other readers aren't blocked after this point. -+ */ -+ downgrade_write(¤t->mm->mmap_lock); -+ read_locked = true; -+ -+ /* Allocate some more pages */ -+ if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, delta) != 0) { -+ res = -ENOMEM; -+ goto out_unlock; -+ } -+ if (reg->cpu_alloc != reg->gpu_alloc) { -+ if (kbase_alloc_phy_pages_helper( -+ reg->gpu_alloc, delta) != 0) { -+ res = -ENOMEM; -+ kbase_free_phy_pages_helper(reg->cpu_alloc, -+ delta); -+ goto out_unlock; -+ } -+ } -+ -+ /* No update required for CPU mappings, that's done on fault. */ -+ -+ /* Update GPU mapping. */ -+ res = kbase_mem_grow_gpu_mapping(kctx, reg, -+ new_pages, old_pages); -+ -+ /* On error free the new pages */ -+ if (res) { -+ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); -+ if (reg->cpu_alloc != reg->gpu_alloc) -+ kbase_free_phy_pages_helper(reg->gpu_alloc, -+ delta); -+ res = -ENOMEM; -+ goto out_unlock; -+ } -+ } else { -+ delta = old_pages - new_pages; -+ -+ /* Update all CPU mapping(s) */ -+ kbase_mem_shrink_cpu_mapping(kctx, reg, -+ new_pages, old_pages); -+ -+ /* Update the GPU mapping */ -+ res = kbase_mem_shrink_gpu_mapping(kctx, reg, -+ new_pages, old_pages); -+ if (res) { -+ res = -ENOMEM; -+ goto out_unlock; -+ } -+ -+ kbase_free_phy_pages_helper(reg->cpu_alloc, delta); -+ if (reg->cpu_alloc != reg->gpu_alloc) -+ kbase_free_phy_pages_helper(reg->gpu_alloc, delta); -+ } -+ -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ if (read_locked) -+ up_read(¤t->mm->mmap_lock); -+ else -+ up_write(¤t->mm->mmap_lock); -+ -+ return res; -+} -+ -+static void kbase_cpu_vm_open(struct vm_area_struct *vma) -+{ -+ struct kbase_cpu_mapping *map = vma->vm_private_data; -+ -+ KBASE_DEBUG_ASSERT(map); -+ KBASE_DEBUG_ASSERT(map->count > 0); -+ /* non-atomic as we're under Linux' mm lock */ -+ map->count++; -+} -+ -+static void kbase_cpu_vm_close(struct vm_area_struct *vma) -+{ -+ struct kbase_cpu_mapping *map = vma->vm_private_data; -+ -+ KBASE_DEBUG_ASSERT(map); -+ KBASE_DEBUG_ASSERT(map->count > 0); -+ -+ /* non-atomic as we're under Linux' mm lock */ -+ if (--map->count) -+ return; -+ -+ KBASE_DEBUG_ASSERT(map->kctx); -+ KBASE_DEBUG_ASSERT(map->alloc); -+ -+ kbase_gpu_vm_lock(map->kctx); -+ -+ if (map->free_on_close) { -+ KBASE_DEBUG_ASSERT((map->region->flags & KBASE_REG_ZONE_MASK) == -+ KBASE_REG_ZONE_SAME_VA); -+ /* Avoid freeing memory on the process death which results in -+ * GPU Page Fault. Memory will be freed in kbase_destroy_context -+ */ -+ if (!(current->flags & PF_EXITING)) -+ kbase_mem_free_region(map->kctx, map->region); -+ } -+ -+ list_del(&map->mappings_list); -+ -+ kbase_gpu_vm_unlock(map->kctx); -+ -+ kbase_mem_phy_alloc_put(map->alloc); -+ kfree(map); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_cpu_vm_close); -+ -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) -+static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) -+{ -+#else -+static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) -+{ -+ struct vm_area_struct *vma = vmf->vma; -+#endif -+ struct kbase_cpu_mapping *map = vma->vm_private_data; -+ pgoff_t rel_pgoff; -+ size_t i; -+ pgoff_t addr; -+ vm_fault_t ret = VM_FAULT_SIGBUS; -+ -+ KBASE_DEBUG_ASSERT(map); -+ KBASE_DEBUG_ASSERT(map->count > 0); -+ KBASE_DEBUG_ASSERT(map->kctx); -+ KBASE_DEBUG_ASSERT(map->alloc); -+ -+ rel_pgoff = vmf->pgoff - map->region->start_pfn; -+ -+ kbase_gpu_vm_lock(map->kctx); -+ if (rel_pgoff >= map->alloc->nents) -+ goto locked_bad_fault; -+ -+ /* Fault on access to DONT_NEED regions */ -+ if (map->alloc->reg && (map->alloc->reg->flags & KBASE_REG_DONT_NEED)) -+ goto locked_bad_fault; -+ -+ /* insert all valid pages from the fault location */ -+ i = rel_pgoff; -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ addr = (pgoff_t)((uintptr_t)vmf->virtual_address >> PAGE_SHIFT); -+#else -+ addr = (pgoff_t)(vmf->address >> PAGE_SHIFT); -+#endif -+ while (i < map->alloc->nents && (addr < vma->vm_end >> PAGE_SHIFT)) { -+ ret = vmf_insert_pfn(vma, addr << PAGE_SHIFT, -+ PFN_DOWN(map->alloc->pages[i])); -+ if (ret != VM_FAULT_NOPAGE) -+ goto locked_bad_fault; -+ -+ i++; addr++; -+ } -+ -+ kbase_gpu_vm_unlock(map->kctx); -+ /* we resolved it, nothing for VM to do */ -+ return VM_FAULT_NOPAGE; -+ -+locked_bad_fault: -+ kbase_gpu_vm_unlock(map->kctx); -+ return ret; -+} -+ -+const struct vm_operations_struct kbase_vm_ops = { -+ .open = kbase_cpu_vm_open, -+ .close = kbase_cpu_vm_close, -+ .fault = kbase_cpu_vm_fault -+}; -+ -+static int kbase_cpu_mmap(struct kbase_va_region *reg, struct vm_area_struct *vma, void *kaddr, size_t nr_pages, unsigned long aligned_offset, int free_on_close) -+{ -+ struct kbase_cpu_mapping *map; -+ phys_addr_t *page_array; -+ int err = 0; -+ int i; -+ -+ map = kzalloc(sizeof(*map), GFP_KERNEL); -+ -+ if (!map) { -+ WARN_ON(1); -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ /* -+ * VM_DONTCOPY - don't make this mapping available in fork'ed processes -+ * VM_DONTEXPAND - disable mremap on this region -+ * VM_IO - disables paging -+ * VM_DONTDUMP - Don't include in core dumps (3.7 only) -+ * VM_MIXEDMAP - Support mixing struct page*s and raw pfns. -+ * This is needed to support using the dedicated and -+ * the OS based memory backends together. -+ */ -+ /* -+ * This will need updating to propagate coherency flags -+ * See MIDBASE-1057 -+ */ -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) -+ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; -+#else -+ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; -+#endif -+ vma->vm_ops = &kbase_vm_ops; -+ vma->vm_private_data = map; -+ -+ page_array = kbase_get_cpu_phy_pages(reg); -+ -+ if (!(reg->flags & KBASE_REG_CPU_CACHED) && -+ (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { -+ /* We can't map vmalloc'd memory uncached. -+ * Other memory will have been returned from -+ * kbase_mem_pool which would be -+ * suitable for mapping uncached. -+ */ -+ BUG_ON(kaddr); -+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); -+ } -+ -+ if (!kaddr) { -+ unsigned long addr = vma->vm_start + aligned_offset; -+ u64 start_off = vma->vm_pgoff - reg->start_pfn + -+ (aligned_offset>>PAGE_SHIFT); -+ -+ vma->vm_flags |= VM_PFNMAP; -+ for (i = 0; i < nr_pages; i++) { -+ unsigned long pfn = PFN_DOWN(page_array[i + start_off]); -+ vm_fault_t ret; -+ -+ ret = vmf_insert_pfn(vma, addr, pfn); -+ if (WARN_ON(ret != VM_FAULT_NOPAGE)) { -+ if (ret == VM_FAULT_OOM) -+ err = -ENOMEM; -+ else -+ err = -EFAULT; -+ break; -+ } -+ -+ addr += PAGE_SIZE; -+ } -+ } else { -+ WARN_ON(aligned_offset); -+ /* MIXEDMAP so we can vfree the kaddr early and not track it after map time */ -+ vma->vm_flags |= VM_MIXEDMAP; -+ /* vmalloc remaping is easy... */ -+ err = remap_vmalloc_range(vma, kaddr, 0); -+ WARN_ON(err); -+ } -+ -+ if (err) { -+ kfree(map); -+ goto out; -+ } -+ -+ map->region = reg; -+ map->free_on_close = free_on_close; -+ map->kctx = reg->kctx; -+ map->alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); -+ map->count = 1; /* start with one ref */ -+ -+ if (reg->flags & KBASE_REG_CPU_CACHED) -+ map->alloc->properties |= KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED; -+ -+ list_add(&map->mappings_list, &map->alloc->mappings); -+ -+ out: -+ return err; -+} -+ -+static int kbase_trace_buffer_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kaddr) -+{ -+ struct kbase_va_region *new_reg; -+ u32 nr_pages; -+ size_t size; -+ int err = 0; -+ u32 *tb; -+ int owns_tb = 1; -+ -+ dev_dbg(kctx->kbdev->dev, "in %s\n", __func__); -+ size = (vma->vm_end - vma->vm_start); -+ nr_pages = size >> PAGE_SHIFT; -+ -+ if (!kctx->jctx.tb) { -+ KBASE_DEBUG_ASSERT(0 != size); -+ tb = vmalloc_user(size); -+ -+ if (NULL == tb) { -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ err = kbase_device_trace_buffer_install(kctx, tb, size); -+ if (err) { -+ vfree(tb); -+ goto out; -+ } -+ } else { -+ err = -EINVAL; -+ goto out; -+ } -+ -+ *kaddr = kctx->jctx.tb; -+ -+ new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); -+ if (!new_reg) { -+ err = -ENOMEM; -+ WARN_ON(1); -+ goto out_no_region; -+ } -+ -+ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_TB); -+ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { -+ err = -ENOMEM; -+ new_reg->cpu_alloc = NULL; -+ WARN_ON(1); -+ goto out_no_alloc; -+ } -+ -+ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); -+ -+ new_reg->cpu_alloc->imported.kctx = kctx; -+ new_reg->flags &= ~KBASE_REG_FREE; -+ new_reg->flags |= KBASE_REG_CPU_CACHED; -+ -+ /* alloc now owns the tb */ -+ owns_tb = 0; -+ -+ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { -+ err = -ENOMEM; -+ WARN_ON(1); -+ goto out_no_va_region; -+ } -+ -+ *reg = new_reg; -+ -+ /* map read only, noexec */ -+ vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); -+ /* the rest of the flags is added by the cpu_mmap handler */ -+ -+ dev_dbg(kctx->kbdev->dev, "%s done\n", __func__); -+ return 0; -+ -+out_no_va_region: -+out_no_alloc: -+ kbase_free_alloced_region(new_reg); -+out_no_region: -+ if (owns_tb) { -+ kbase_device_trace_buffer_uninstall(kctx); -+ vfree(tb); -+ } -+out: -+ return err; -+} -+ -+static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, struct kbase_va_region **const reg, void **const kmap_addr) -+{ -+ struct kbase_va_region *new_reg; -+ void *kaddr; -+ u32 nr_pages; -+ size_t size; -+ int err = 0; -+ -+ dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n"); -+ size = (vma->vm_end - vma->vm_start); -+ nr_pages = size >> PAGE_SHIFT; -+ -+ kaddr = kbase_mmu_dump(kctx, nr_pages); -+ -+ if (!kaddr) { -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ new_reg = kbase_alloc_free_region(kctx, 0, nr_pages, KBASE_REG_ZONE_SAME_VA); -+ if (!new_reg) { -+ err = -ENOMEM; -+ WARN_ON(1); -+ goto out; -+ } -+ -+ new_reg->cpu_alloc = kbase_alloc_create(0, KBASE_MEM_TYPE_RAW); -+ if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { -+ err = -ENOMEM; -+ new_reg->cpu_alloc = NULL; -+ WARN_ON(1); -+ goto out_no_alloc; -+ } -+ -+ new_reg->gpu_alloc = kbase_mem_phy_alloc_get(new_reg->cpu_alloc); -+ -+ new_reg->flags &= ~KBASE_REG_FREE; -+ new_reg->flags |= KBASE_REG_CPU_CACHED; -+ if (kbase_add_va_region(kctx, new_reg, vma->vm_start, nr_pages, 1) != 0) { -+ err = -ENOMEM; -+ WARN_ON(1); -+ goto out_va_region; -+ } -+ -+ *kmap_addr = kaddr; -+ *reg = new_reg; -+ -+ dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n"); -+ return 0; -+ -+out_no_alloc: -+out_va_region: -+ kbase_free_alloced_region(new_reg); -+out: -+ return err; -+} -+ -+ -+void kbase_os_mem_map_lock(struct kbase_context *kctx) -+{ -+ struct mm_struct *mm = current->mm; -+ (void)kctx; -+ down_read(&mm->mmap_lock); -+} -+ -+void kbase_os_mem_map_unlock(struct kbase_context *kctx) -+{ -+ struct mm_struct *mm = current->mm; -+ (void)kctx; -+ up_read(&mm->mmap_lock); -+} -+ -+static int kbasep_reg_mmap(struct kbase_context *kctx, -+ struct vm_area_struct *vma, -+ struct kbase_va_region **regm, -+ size_t *nr_pages, size_t *aligned_offset) -+ -+{ -+ int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); -+ struct kbase_va_region *reg; -+ int err = 0; -+ -+ *aligned_offset = 0; -+ -+ dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n"); -+ -+ /* SAME_VA stuff, fetch the right region */ -+ reg = kctx->pending_regions[cookie]; -+ if (!reg) { -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ if ((reg->flags & KBASE_REG_GPU_NX) && (reg->nr_pages != *nr_pages)) { -+ /* incorrect mmap size */ -+ /* leave the cookie for a potential later -+ * mapping, or to be reclaimed later when the -+ * context is freed */ -+ err = -ENOMEM; -+ goto out; -+ } -+ -+ if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || -+ (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { -+ /* VM flags inconsistent with region flags */ -+ err = -EPERM; -+ dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", -+ __FILE__, __LINE__); -+ goto out; -+ } -+ -+ /* adjust down nr_pages to what we have physically */ -+ *nr_pages = kbase_reg_current_backed_size(reg); -+ -+ if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, -+ reg->nr_pages, 1) != 0) { -+ dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); -+ /* Unable to map in GPU space. */ -+ WARN_ON(1); -+ err = -ENOMEM; -+ goto out; -+ } -+ /* no need for the cookie anymore */ -+ kctx->pending_regions[cookie] = NULL; -+ kctx->cookies |= (1UL << cookie); -+ -+ /* -+ * Overwrite the offset with the region start_pfn, so we effectively -+ * map from offset 0 in the region. However subtract the aligned -+ * offset so that when user space trims the mapping the beginning of -+ * the trimmed VMA has the correct vm_pgoff; -+ */ -+ vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); -+out: -+ *regm = reg; -+ dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n"); -+ -+ return err; -+} -+ -+int kbase_mmap(struct file *file, struct vm_area_struct *vma) -+{ -+ struct kbase_context *kctx = file->private_data; -+ struct kbase_va_region *reg = NULL; -+ void *kaddr = NULL; -+ size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; -+ int err = 0; -+ int free_on_close = 0; -+ struct device *dev = kctx->kbdev->dev; -+ size_t aligned_offset = 0; -+ -+ dev_dbg(dev, "kbase_mmap\n"); -+ -+ /* strip away corresponding VM_MAY% flags to the VM_% flags requested */ -+ vma->vm_flags &= ~((vma->vm_flags & (VM_READ | VM_WRITE)) << 4); -+ -+ if (0 == nr_pages) { -+ err = -EINVAL; -+ goto out; -+ } -+ -+ if (!(vma->vm_flags & VM_SHARED)) { -+ err = -EINVAL; -+ goto out; -+ } -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MAP_TRACKING_HANDLE)) { -+ /* The non-mapped tracking helper page */ -+ err = kbase_tracking_page_setup(kctx, vma); -+ goto out_unlock; -+ } -+ -+ /* if not the MTP, verify that the MTP has been mapped */ -+ rcu_read_lock(); -+ /* catches both when the special page isn't present or -+ * when we've forked */ -+ if (rcu_dereference(kctx->process_mm) != current->mm) { -+ err = -EINVAL; -+ rcu_read_unlock(); -+ goto out_unlock; -+ } -+ rcu_read_unlock(); -+ -+ switch (vma->vm_pgoff) { -+ case PFN_DOWN(BASEP_MEM_INVALID_HANDLE): -+ case PFN_DOWN(BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE): -+ /* Illegal handle for direct map */ -+ err = -EINVAL; -+ goto out_unlock; -+ case PFN_DOWN(BASE_MEM_TRACE_BUFFER_HANDLE): -+ err = kbase_trace_buffer_mmap(kctx, vma, ®, &kaddr); -+ if (0 != err) -+ goto out_unlock; -+ dev_dbg(dev, "kbase_trace_buffer_mmap ok\n"); -+ /* free the region on munmap */ -+ free_on_close = 1; -+ break; -+ case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): -+ /* MMU dump */ -+ err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); -+ if (0 != err) -+ goto out_unlock; -+ /* free the region on munmap */ -+ free_on_close = 1; -+ break; -+ case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... -+ PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { -+ err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, -+ &aligned_offset); -+ if (0 != err) -+ goto out_unlock; -+ /* free the region on munmap */ -+ free_on_close = 1; -+ break; -+ } -+ default: { -+ reg = kbase_region_tracker_find_region_enclosing_address(kctx, -+ (u64)vma->vm_pgoff << PAGE_SHIFT); -+ -+ if (reg && !(reg->flags & KBASE_REG_FREE)) { -+ /* will this mapping overflow the size of the region? */ -+ if (nr_pages > (reg->nr_pages - -+ (vma->vm_pgoff - reg->start_pfn))) { -+ err = -ENOMEM; -+ goto out_unlock; -+ } -+ -+ if ((vma->vm_flags & VM_READ && -+ !(reg->flags & KBASE_REG_CPU_RD)) || -+ (vma->vm_flags & VM_WRITE && -+ !(reg->flags & KBASE_REG_CPU_WR))) { -+ /* VM flags inconsistent with region flags */ -+ err = -EPERM; -+ dev_err(dev, "%s:%d inconsistent VM flags\n", -+ __FILE__, __LINE__); -+ goto out_unlock; -+ } -+ -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ if (KBASE_MEM_TYPE_IMPORTED_UMM == -+ reg->cpu_alloc->type) { -+ err = dma_buf_mmap( -+ reg->cpu_alloc->imported.umm.dma_buf, -+ vma, vma->vm_pgoff - reg->start_pfn); -+ goto out_unlock; -+ } -+#endif /* CONFIG_DMA_SHARED_BUFFER */ -+ -+ /* limit what we map to the amount currently backed */ -+ if (reg->cpu_alloc->nents < (vma->vm_pgoff - reg->start_pfn + nr_pages)) { -+ if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) -+ nr_pages = 0; -+ else -+ nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); -+ } -+ } else { -+ err = -ENOMEM; -+ goto out_unlock; -+ } -+ } /* default */ -+ } /* switch */ -+ -+ err = kbase_cpu_mmap(reg, vma, kaddr, nr_pages, aligned_offset, free_on_close); -+ -+ if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { -+ /* MMU dump - userspace should now have a reference on -+ * the pages, so we can now free the kernel mapping */ -+ vfree(kaddr); -+ } -+ -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+out: -+ if (err) -+ dev_err(dev, "mmap failed %d\n", err); -+ -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mmap); -+ -+void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, -+ unsigned long prot_request, struct kbase_vmap_struct *map) -+{ -+ struct kbase_va_region *reg; -+ unsigned long page_index; -+ unsigned int offset = gpu_addr & ~PAGE_MASK; -+ size_t page_count = PFN_UP(offset + size); -+ phys_addr_t *page_array; -+ struct page **pages; -+ void *cpu_addr = NULL; -+ pgprot_t prot; -+ size_t i; -+ bool sync_needed; -+ -+ if (!size || !map) -+ return NULL; -+ -+ /* check if page_count calculation will wrap */ -+ if (size > ((size_t)-1 / PAGE_SIZE)) -+ return NULL; -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); -+ if (!reg || (reg->flags & KBASE_REG_FREE)) -+ goto out_unlock; -+ -+ page_index = (gpu_addr >> PAGE_SHIFT) - reg->start_pfn; -+ -+ /* check if page_index + page_count will wrap */ -+ if (-1UL - page_count < page_index) -+ goto out_unlock; -+ -+ if (page_index + page_count > kbase_reg_current_backed_size(reg)) -+ goto out_unlock; -+ -+ if (reg->flags & KBASE_REG_DONT_NEED) -+ goto out_unlock; -+ -+ /* check access permissions can be satisfied -+ * Intended only for checking KBASE_REG_{CPU,GPU}_{RD,WR} */ -+ if ((reg->flags & prot_request) != prot_request) -+ goto out_unlock; -+ -+ page_array = kbase_get_cpu_phy_pages(reg); -+ if (!page_array) -+ goto out_unlock; -+ -+ pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); -+ if (!pages) -+ goto out_unlock; -+ -+ for (i = 0; i < page_count; i++) -+ pages[i] = pfn_to_page(PFN_DOWN(page_array[page_index + i])); -+ -+ prot = PAGE_KERNEL; -+ if (!(reg->flags & KBASE_REG_CPU_CACHED)) { -+ /* Map uncached */ -+ prot = pgprot_writecombine(prot); -+ } -+ /* Note: enforcing a RO prot_request onto prot is not done, since: -+ * - CPU-arch-specific integration required -+ * - kbase_vmap() requires no access checks to be made/enforced */ -+ -+ cpu_addr = vmap(pages, page_count, VM_MAP, prot); -+ -+ kfree(pages); -+ -+ if (!cpu_addr) -+ goto out_unlock; -+ -+ map->gpu_addr = gpu_addr; -+ map->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); -+ map->cpu_pages = &kbase_get_cpu_phy_pages(reg)[page_index]; -+ map->gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ map->gpu_pages = &kbase_get_gpu_phy_pages(reg)[page_index]; -+ map->addr = (void *)((uintptr_t)cpu_addr + offset); -+ map->size = size; -+ map->is_cached = (reg->flags & KBASE_REG_CPU_CACHED) != 0; -+ sync_needed = map->is_cached; -+ -+#ifdef CONFIG_MALI_COH_KERN -+ /* kernel can use coherent memory if supported */ -+ if (kctx->kbdev->system_coherency == COHERENCY_ACE) -+ sync_needed = false; -+#endif -+ -+ if (sync_needed) { -+ /* Sync first page */ -+ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); -+ phys_addr_t cpu_pa = map->cpu_pages[0]; -+ phys_addr_t gpu_pa = map->gpu_pages[0]; -+ -+ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, -+ KBASE_SYNC_TO_CPU); -+ -+ /* Sync middle pages (if any) */ -+ for (i = 1; page_count > 2 && i < page_count - 1; i++) { -+ cpu_pa = map->cpu_pages[i]; -+ gpu_pa = map->gpu_pages[i]; -+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, -+ KBASE_SYNC_TO_CPU); -+ } -+ -+ /* Sync last page (if any) */ -+ if (page_count > 1) { -+ cpu_pa = map->cpu_pages[page_count - 1]; -+ gpu_pa = map->gpu_pages[page_count - 1]; -+ sz = ((offset + size - 1) & ~PAGE_MASK) + 1; -+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, -+ KBASE_SYNC_TO_CPU); -+ } -+ } -+ kbase_gpu_vm_unlock(kctx); -+ -+ return map->addr; -+ -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return NULL; -+} -+ -+void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, -+ struct kbase_vmap_struct *map) -+{ -+ /* 0 is specified for prot_request to indicate no access checks should -+ * be made. -+ * -+ * As mentioned in kbase_vmap_prot() this means that a kernel-side -+ * CPU-RO mapping is not enforced to allow this to work */ -+ return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); -+} -+KBASE_EXPORT_TEST_API(kbase_vmap); -+ -+void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map) -+{ -+ void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); -+ bool sync_needed = map->is_cached; -+ vunmap(addr); -+#ifdef CONFIG_MALI_COH_KERN -+ /* kernel can use coherent memory if supported */ -+ if (kctx->kbdev->system_coherency == COHERENCY_ACE) -+ sync_needed = false; -+#endif -+ if (sync_needed) { -+ off_t offset = (uintptr_t)map->addr & ~PAGE_MASK; -+ size_t size = map->size; -+ size_t page_count = PFN_UP(offset + size); -+ size_t i; -+ -+ /* Sync first page */ -+ size_t sz = MIN(((size_t) PAGE_SIZE - offset), size); -+ phys_addr_t cpu_pa = map->cpu_pages[0]; -+ phys_addr_t gpu_pa = map->gpu_pages[0]; -+ -+ kbase_sync_single(kctx, cpu_pa, gpu_pa, offset, sz, -+ KBASE_SYNC_TO_DEVICE); -+ -+ /* Sync middle pages (if any) */ -+ for (i = 1; page_count > 2 && i < page_count - 1; i++) { -+ cpu_pa = map->cpu_pages[i]; -+ gpu_pa = map->gpu_pages[i]; -+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, PAGE_SIZE, -+ KBASE_SYNC_TO_DEVICE); -+ } -+ -+ /* Sync last page (if any) */ -+ if (page_count > 1) { -+ cpu_pa = map->cpu_pages[page_count - 1]; -+ gpu_pa = map->gpu_pages[page_count - 1]; -+ sz = ((offset + size - 1) & ~PAGE_MASK) + 1; -+ kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, -+ KBASE_SYNC_TO_DEVICE); -+ } -+ } -+ map->gpu_addr = 0; -+ map->cpu_alloc = kbase_mem_phy_alloc_put(map->cpu_alloc); -+ map->gpu_alloc = kbase_mem_phy_alloc_put(map->gpu_alloc); -+ map->cpu_pages = NULL; -+ map->gpu_pages = NULL; -+ map->addr = NULL; -+ map->size = 0; -+ map->is_cached = false; -+} -+KBASE_EXPORT_TEST_API(kbase_vunmap); -+ -+void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages) -+{ -+ struct mm_struct *mm; -+ -+ rcu_read_lock(); -+ mm = rcu_dereference(kctx->process_mm); -+ if (mm) { -+ atomic_add(pages, &kctx->nonmapped_pages); -+#ifdef SPLIT_RSS_COUNTING -+ add_mm_counter(mm, MM_FILEPAGES, pages); -+#else -+ spin_lock(&mm->page_table_lock); -+ add_mm_counter(mm, MM_FILEPAGES, pages); -+ spin_unlock(&mm->page_table_lock); -+#endif -+ } -+ rcu_read_unlock(); -+} -+ -+static void kbasep_os_process_page_usage_drain(struct kbase_context *kctx) -+{ -+ int pages; -+ struct mm_struct *mm; -+ -+ spin_lock(&kctx->mm_update_lock); -+ mm = rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock)); -+ if (!mm) { -+ spin_unlock(&kctx->mm_update_lock); -+ return; -+ } -+ -+ rcu_assign_pointer(kctx->process_mm, NULL); -+ spin_unlock(&kctx->mm_update_lock); -+ synchronize_rcu(); -+ -+ pages = atomic_xchg(&kctx->nonmapped_pages, 0); -+#ifdef SPLIT_RSS_COUNTING -+ add_mm_counter(mm, MM_FILEPAGES, -pages); -+#else -+ spin_lock(&mm->page_table_lock); -+ add_mm_counter(mm, MM_FILEPAGES, -pages); -+ spin_unlock(&mm->page_table_lock); -+#endif -+} -+ -+static void kbase_special_vm_close(struct vm_area_struct *vma) -+{ -+ struct kbase_context *kctx; -+ -+ kctx = vma->vm_private_data; -+ kbasep_os_process_page_usage_drain(kctx); -+} -+ -+static const struct vm_operations_struct kbase_vm_special_ops = { -+ .close = kbase_special_vm_close, -+}; -+ -+static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma) -+{ -+ /* check that this is the only tracking page */ -+ spin_lock(&kctx->mm_update_lock); -+ if (rcu_dereference_protected(kctx->process_mm, lockdep_is_held(&kctx->mm_update_lock))) { -+ spin_unlock(&kctx->mm_update_lock); -+ return -EFAULT; -+ } -+ -+ rcu_assign_pointer(kctx->process_mm, current->mm); -+ -+ spin_unlock(&kctx->mm_update_lock); -+ -+ /* no real access */ -+ vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) -+ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; -+#else -+ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; -+#endif -+ vma->vm_ops = &kbase_vm_special_ops; -+ vma->vm_private_data = kctx; -+ -+ return 0; -+} -+void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle) -+{ -+ int i; -+ int res; -+ void *va; -+ dma_addr_t dma_pa; -+ struct kbase_va_region *reg; -+ phys_addr_t *page_array; -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+ unsigned long attrs = DMA_ATTR_WRITE_COMBINE; -+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+ DEFINE_DMA_ATTRS(attrs); -+#endif -+ -+ u32 pages = ((size - 1) >> PAGE_SHIFT) + 1; -+ u32 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR | -+ BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR; -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(0 != size); -+ KBASE_DEBUG_ASSERT(0 != pages); -+ -+ if (size == 0) -+ goto err; -+ -+ /* All the alloc calls return zeroed memory */ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+ va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, -+ attrs); -+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); -+ va = dma_alloc_attrs(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL, -+ &attrs); -+#else -+ va = dma_alloc_writecombine(kctx->kbdev->dev, size, &dma_pa, GFP_KERNEL); -+#endif -+ if (!va) -+ goto err; -+ -+ /* Store the state so we can free it later. */ -+ handle->cpu_va = va; -+ handle->dma_pa = dma_pa; -+ handle->size = size; -+ -+ -+ reg = kbase_alloc_free_region(kctx, 0, pages, KBASE_REG_ZONE_SAME_VA); -+ if (!reg) -+ goto no_reg; -+ -+ reg->flags &= ~KBASE_REG_FREE; -+ if (kbase_update_region_flags(kctx, reg, flags) != 0) -+ goto invalid_flags; -+ -+ reg->cpu_alloc = kbase_alloc_create(pages, KBASE_MEM_TYPE_RAW); -+ if (IS_ERR_OR_NULL(reg->cpu_alloc)) -+ goto no_alloc; -+ -+ reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); -+ -+ page_array = kbase_get_cpu_phy_pages(reg); -+ -+ for (i = 0; i < pages; i++) -+ page_array[i] = dma_pa + (i << PAGE_SHIFT); -+ -+ reg->cpu_alloc->nents = pages; -+ -+ kbase_gpu_vm_lock(kctx); -+ res = kbase_gpu_mmap(kctx, reg, (uintptr_t) va, pages, 1); -+ kbase_gpu_vm_unlock(kctx); -+ if (res) -+ goto no_mmap; -+ -+ return va; -+ -+no_mmap: -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+no_alloc: -+invalid_flags: -+ kfree(reg); -+no_reg: -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+ dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, attrs); -+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+ dma_free_attrs(kctx->kbdev->dev, size, va, dma_pa, &attrs); -+#else -+ dma_free_writecombine(kctx->kbdev->dev, size, va, dma_pa); -+#endif -+err: -+ return NULL; -+} -+KBASE_EXPORT_SYMBOL(kbase_va_alloc); -+ -+void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle) -+{ -+ struct kbase_va_region *reg; -+ int err; -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ -+ (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+ DEFINE_DMA_ATTRS(attrs); -+#endif -+ -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(handle->cpu_va != NULL); -+ -+ kbase_gpu_vm_lock(kctx); -+ reg = kbase_region_tracker_find_region_base_address(kctx, (uintptr_t)handle->cpu_va); -+ KBASE_DEBUG_ASSERT(reg); -+ err = kbase_gpu_munmap(kctx, reg); -+ kbase_gpu_vm_unlock(kctx); -+ KBASE_DEBUG_ASSERT(!err); -+ -+ kbase_mem_phy_alloc_put(reg->cpu_alloc); -+ kbase_mem_phy_alloc_put(reg->gpu_alloc); -+ kfree(reg); -+ -+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+ dma_free_attrs(kctx->kbdev->dev, handle->size, -+ handle->cpu_va, handle->dma_pa, DMA_ATTR_WRITE_COMBINE); -+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+ dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); -+ dma_free_attrs(kctx->kbdev->dev, handle->size, -+ handle->cpu_va, handle->dma_pa, &attrs); -+#else -+ dma_free_writecombine(kctx->kbdev->dev, handle->size, -+ handle->cpu_va, handle->dma_pa); -+#endif -+} -+KBASE_EXPORT_SYMBOL(kbase_va_free); -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h -new file mode 100755 -index 000000000..33b3554f9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h -@@ -0,0 +1,231 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010, 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_mem_linux.h -+ * Base kernel memory APIs, Linux implementation. -+ */ -+ -+#ifndef _KBASE_MEM_LINUX_H_ -+#define _KBASE_MEM_LINUX_H_ -+ -+/** A HWC dump mapping */ -+struct kbase_hwc_dma_mapping { -+ void *cpu_va; -+ dma_addr_t dma_pa; -+ size_t size; -+}; -+ -+struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, -+ u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, -+ u64 *gpu_va); -+int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, int query, u64 *const pages); -+int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, -+ void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, -+ u64 *flags); -+u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); -+int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask); -+ -+/** -+ * kbase_mem_commit - Change the physical backing size of a region -+ * -+ * @kctx: The kernel context -+ * @gpu_addr: Handle to the memory region -+ * @new_pages: Number of physical pages to back the region with -+ * -+ * Return: 0 on success or error code -+ */ -+int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages); -+ -+int kbase_mmap(struct file *file, struct vm_area_struct *vma); -+ -+/** -+ * kbase_mem_evictable_init - Initialize the Ephemeral memory the eviction -+ * mechanism. -+ * @kctx: The kbase context to initialize. -+ * -+ * Return: Zero on success or -errno on failure. -+ */ -+int kbase_mem_evictable_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_mem_evictable_deinit - De-initialize the Ephemeral memory eviction -+ * mechanism. -+ * @kctx: The kbase context to de-initialize. -+ */ -+void kbase_mem_evictable_deinit(struct kbase_context *kctx); -+ -+/** -+ * kbase_mem_grow_gpu_mapping - Grow the GPU mapping of an allocation -+ * @kctx: Context the region belongs to -+ * @reg: The GPU region -+ * @new_pages: The number of pages after the grow -+ * @old_pages: The number of pages before the grow -+ * -+ * Return: 0 on success, -errno on error. -+ * -+ * Expand the GPU mapping to encompass the new psychical pages which have -+ * been added to the allocation. -+ * -+ * Note: Caller must be holding the region lock. -+ */ -+int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, -+ struct kbase_va_region *reg, -+ u64 new_pages, u64 old_pages); -+ -+/** -+ * kbase_mem_evictable_make - Make a physical allocation eligible for eviction -+ * @gpu_alloc: The physical allocation to make evictable -+ * -+ * Return: 0 on success, -errno on error. -+ * -+ * Take the provided region and make all the physical pages within it -+ * reclaimable by the kernel, updating the per-process VM stats as well. -+ * Remove any CPU mappings (as these can't be removed in the shrinker callback -+ * as mmap_lock might already be taken) but leave the GPU mapping intact as -+ * and until the shrinker reclaims the allocation. -+ * -+ * Note: Must be called with the region lock of the containing context. -+ */ -+int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc); -+ -+/** -+ * kbase_mem_evictable_unmake - Remove a physical allocations eligibility for -+ * eviction. -+ * @alloc: The physical allocation to remove eviction eligibility from. -+ * -+ * Return: True if the allocation had its backing restored and false if -+ * it hasn't. -+ * -+ * Make the physical pages in the region no longer reclaimable and update the -+ * per-process stats, if the shrinker has already evicted the memory then -+ * re-allocate it if the region is still alive. -+ * -+ * Note: Must be called with the region lock of the containing context. -+ */ -+bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *alloc); -+ -+struct kbase_vmap_struct { -+ u64 gpu_addr; -+ struct kbase_mem_phy_alloc *cpu_alloc; -+ struct kbase_mem_phy_alloc *gpu_alloc; -+ phys_addr_t *cpu_pages; -+ phys_addr_t *gpu_pages; -+ void *addr; -+ size_t size; -+ bool is_cached; -+}; -+ -+ -+/** -+ * kbase_vmap_prot - Map a GPU VA range into the kernel safely, only if the -+ * requested access permissions are supported -+ * @kctx: Context the VA range belongs to -+ * @gpu_addr: Start address of VA range -+ * @size: Size of VA range -+ * @prot_request: Flags indicating how the caller will then access the memory -+ * @map: Structure to be given to kbase_vunmap() on freeing -+ * -+ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error -+ * -+ * Map a GPU VA Range into the kernel. The VA range must be contained within a -+ * GPU memory region. Appropriate CPU cache-flushing operations are made as -+ * required, dependent on the CPU mapping for the memory region. -+ * -+ * This is safer than using kmap() on the pages directly, -+ * because the pages here are refcounted to prevent freeing (and hence reuse -+ * elsewhere in the system) until an kbase_vunmap() -+ * -+ * The flags in @prot_request should use KBASE_REG_{CPU,GPU}_{RD,WR}, to check -+ * whether the region should allow the intended access, and return an error if -+ * disallowed. This is essential for security of imported memory, particularly -+ * a user buf from SHM mapped into the process as RO. In that case, write -+ * access must be checked if the intention is for kernel to write to the -+ * memory. -+ * -+ * The checks are also there to help catch access errors on memory where -+ * security is not a concern: imported memory that is always RW, and memory -+ * that was allocated and owned by the process attached to @kctx. In this case, -+ * it helps to identify memory that was was mapped with the wrong access type. -+ * -+ * Note: KBASE_REG_GPU_{RD,WR} flags are currently supported for legacy cases -+ * where either the security of memory is solely dependent on those flags, or -+ * when userspace code was expecting only the GPU to access the memory (e.g. HW -+ * workarounds). -+ * -+ */ -+void *kbase_vmap_prot(struct kbase_context *kctx, u64 gpu_addr, size_t size, -+ unsigned long prot_request, struct kbase_vmap_struct *map); -+ -+/** -+ * kbase_vmap - Map a GPU VA range into the kernel safely -+ * @kctx: Context the VA range belongs to -+ * @gpu_addr: Start address of VA range -+ * @size: Size of VA range -+ * @map: Structure to be given to kbase_vunmap() on freeing -+ * -+ * Return: Kernel-accessible CPU pointer to the VA range, or NULL on error -+ * -+ * Map a GPU VA Range into the kernel. The VA range must be contained within a -+ * GPU memory region. Appropriate CPU cache-flushing operations are made as -+ * required, dependent on the CPU mapping for the memory region. -+ * -+ * This is safer than using kmap() on the pages directly, -+ * because the pages here are refcounted to prevent freeing (and hence reuse -+ * elsewhere in the system) until an kbase_vunmap() -+ * -+ * kbase_vmap_prot() should be used in preference, since kbase_vmap() makes no -+ * checks to ensure the security of e.g. imported user bufs from RO SHM. -+ */ -+void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, -+ struct kbase_vmap_struct *map); -+ -+/** -+ * kbase_vunmap - Unmap a GPU VA range from the kernel -+ * @kctx: Context the VA range belongs to -+ * @map: Structure describing the mapping from the corresponding kbase_vmap() -+ * call -+ * -+ * Unmaps a GPU VA range from the kernel, given its @map structure obtained -+ * from kbase_vmap(). Appropriate CPU cache-flushing operations are made as -+ * required, dependent on the CPU mapping for the memory region. -+ * -+ * The reference taken on pages during kbase_vmap() is released. -+ */ -+void kbase_vunmap(struct kbase_context *kctx, struct kbase_vmap_struct *map); -+ -+/** @brief Allocate memory from kernel space and map it onto the GPU -+ * -+ * @param kctx The context used for the allocation/mapping -+ * @param size The size of the allocation in bytes -+ * @param handle An opaque structure used to contain the state needed to free the memory -+ * @return the VA for kernel space and GPU MMU -+ */ -+void *kbase_va_alloc(struct kbase_context *kctx, u32 size, struct kbase_hwc_dma_mapping *handle); -+ -+/** @brief Free/unmap memory allocated by kbase_va_alloc -+ * -+ * @param kctx The context used for the allocation/mapping -+ * @param handle An opaque structure returned by the kbase_va_alloc function. -+ */ -+void kbase_va_free(struct kbase_context *kctx, struct kbase_hwc_dma_mapping *handle); -+ -+extern const struct vm_operations_struct kbase_vm_ops; -+ -+#endif /* _KBASE_MEM_LINUX_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h -new file mode 100755 -index 000000000..9725fd3f0 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h -@@ -0,0 +1,45 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_MEM_LOWLEVEL_H -+#define _KBASE_MEM_LOWLEVEL_H -+ -+#ifndef _KBASE_H_ -+#error "Don't include this file directly, use mali_kbase.h instead" -+#endif -+ -+#include -+ -+/** -+ * @brief Flags for kbase_phy_allocator_pages_alloc -+ */ -+#define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ -+#define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ -+#define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ -+ -+#define KBASE_PHY_PAGES_SUPPORTED_FLAGS (KBASE_PHY_PAGES_FLAG_DEFAULT|KBASE_PHY_PAGES_FLAG_CLEAR|KBASE_PHY_PAGES_FLAG_POISON) -+ -+#define KBASE_PHY_PAGES_POISON_VALUE 0xFD /** Value to fill the memory with when KBASE_PHY_PAGES_FLAG_POISON is set */ -+ -+enum kbase_sync_type { -+ KBASE_SYNC_TO_CPU, -+ KBASE_SYNC_TO_DEVICE -+}; -+ -+#endif /* _KBASE_LOWLEVEL_H */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c b/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c -new file mode 100755 -index 000000000..a8269940a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c -@@ -0,0 +1,569 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define pool_dbg(pool, format, ...) \ -+ dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ -+ (pool->next_pool) ? "kctx" : "kbdev", \ -+ kbase_mem_pool_size(pool), \ -+ kbase_mem_pool_max_size(pool), \ -+ ##__VA_ARGS__) -+ -+#define NOT_DIRTY false -+#define NOT_RECLAIMED false -+ -+static inline void kbase_mem_pool_lock(struct kbase_mem_pool *pool) -+{ -+ spin_lock(&pool->pool_lock); -+} -+ -+static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) -+{ -+ spin_unlock(&pool->pool_lock); -+} -+ -+static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) -+{ -+ ssize_t max_size = kbase_mem_pool_max_size(pool); -+ ssize_t cur_size = kbase_mem_pool_size(pool); -+ -+ return max(max_size - cur_size, (ssize_t)0); -+} -+ -+static bool kbase_mem_pool_is_full(struct kbase_mem_pool *pool) -+{ -+ return kbase_mem_pool_size(pool) >= kbase_mem_pool_max_size(pool); -+} -+ -+static bool kbase_mem_pool_is_empty(struct kbase_mem_pool *pool) -+{ -+ return kbase_mem_pool_size(pool) == 0; -+} -+ -+static void kbase_mem_pool_add_locked(struct kbase_mem_pool *pool, -+ struct page *p) -+{ -+ lockdep_assert_held(&pool->pool_lock); -+ -+ list_add(&p->lru, &pool->page_list); -+ pool->cur_size++; -+ -+ pool_dbg(pool, "added page\n"); -+} -+ -+static void kbase_mem_pool_add(struct kbase_mem_pool *pool, struct page *p) -+{ -+ kbase_mem_pool_lock(pool); -+ kbase_mem_pool_add_locked(pool, p); -+ kbase_mem_pool_unlock(pool); -+} -+ -+static void kbase_mem_pool_add_list_locked(struct kbase_mem_pool *pool, -+ struct list_head *page_list, size_t nr_pages) -+{ -+ lockdep_assert_held(&pool->pool_lock); -+ -+ list_splice(page_list, &pool->page_list); -+ pool->cur_size += nr_pages; -+ -+ pool_dbg(pool, "added %zu pages\n", nr_pages); -+} -+ -+static void kbase_mem_pool_add_list(struct kbase_mem_pool *pool, -+ struct list_head *page_list, size_t nr_pages) -+{ -+ kbase_mem_pool_lock(pool); -+ kbase_mem_pool_add_list_locked(pool, page_list, nr_pages); -+ kbase_mem_pool_unlock(pool); -+} -+ -+static struct page *kbase_mem_pool_remove_locked(struct kbase_mem_pool *pool) -+{ -+ struct page *p; -+ -+ lockdep_assert_held(&pool->pool_lock); -+ -+ if (kbase_mem_pool_is_empty(pool)) -+ return NULL; -+ -+ p = list_first_entry(&pool->page_list, struct page, lru); -+ list_del_init(&p->lru); -+ pool->cur_size--; -+ -+ pool_dbg(pool, "removed page\n"); -+ -+ return p; -+} -+ -+static struct page *kbase_mem_pool_remove(struct kbase_mem_pool *pool) -+{ -+ struct page *p; -+ -+ kbase_mem_pool_lock(pool); -+ p = kbase_mem_pool_remove_locked(pool); -+ kbase_mem_pool_unlock(pool); -+ -+ return p; -+} -+ -+static void kbase_mem_pool_sync_page(struct kbase_mem_pool *pool, -+ struct page *p) -+{ -+ struct device *dev = pool->kbdev->dev; -+ -+ dma_sync_single_for_device(dev, kbase_dma_addr(p), -+ PAGE_SIZE, DMA_BIDIRECTIONAL); -+} -+ -+static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, -+ struct page *p) -+{ -+ clear_highpage(p); -+ kbase_mem_pool_sync_page(pool, p); -+} -+ -+static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, -+ struct page *p) -+{ -+ /* Zero page before spilling */ -+ kbase_mem_pool_zero_page(next_pool, p); -+ -+ kbase_mem_pool_add(next_pool, p); -+} -+ -+struct page *kbase_mem_alloc_page(struct kbase_device *kbdev) -+{ -+ struct page *p; -+ gfp_t gfp; -+ struct device *dev = kbdev->dev; -+ dma_addr_t dma_addr; -+ -+#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ -+ LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) -+ /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ -+ gfp = GFP_USER | __GFP_ZERO; -+#else -+ gfp = GFP_HIGHUSER | __GFP_ZERO; -+#endif -+ -+ if (current->flags & PF_KTHREAD) { -+ /* Don't trigger OOM killer from kernel threads, e.g. when -+ * growing memory on GPU page fault */ -+ gfp |= __GFP_NORETRY; -+ } -+ -+ p = alloc_page(gfp); -+ if (!p) -+ return NULL; -+ -+ dma_addr = dma_map_page(dev, p, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); -+ if (dma_mapping_error(dev, dma_addr)) { -+ __free_page(p); -+ return NULL; -+ } -+ -+ WARN_ON(dma_addr != page_to_phys(p)); -+ -+ kbase_set_dma_addr(p, dma_addr); -+ -+ return p; -+} -+ -+static void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, -+ struct page *p) -+{ -+ struct device *dev = pool->kbdev->dev; -+ dma_addr_t dma_addr = kbase_dma_addr(p); -+ -+ dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); -+ kbase_clear_dma_addr(p); -+ __free_page(p); -+ -+ pool_dbg(pool, "freed page to kernel\n"); -+} -+ -+static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, -+ size_t nr_to_shrink) -+{ -+ struct page *p; -+ size_t i; -+ -+ lockdep_assert_held(&pool->pool_lock); -+ -+ for (i = 0; i < nr_to_shrink && !kbase_mem_pool_is_empty(pool); i++) { -+ p = kbase_mem_pool_remove_locked(pool); -+ kbase_mem_pool_free_page(pool, p); -+ } -+ -+ return i; -+} -+ -+static size_t kbase_mem_pool_shrink(struct kbase_mem_pool *pool, -+ size_t nr_to_shrink) -+{ -+ size_t nr_freed; -+ -+ kbase_mem_pool_lock(pool); -+ nr_freed = kbase_mem_pool_shrink_locked(pool, nr_to_shrink); -+ kbase_mem_pool_unlock(pool); -+ -+ return nr_freed; -+} -+ -+int kbase_mem_pool_grow(struct kbase_mem_pool *pool, -+ size_t nr_to_grow) -+{ -+ struct page *p; -+ size_t i; -+ -+ for (i = 0; i < nr_to_grow; i++) { -+ p = kbase_mem_alloc_page(pool->kbdev); -+ if (!p) -+ return -ENOMEM; -+ kbase_mem_pool_add(pool, p); -+ } -+ -+ return 0; -+} -+ -+void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) -+{ -+ size_t cur_size; -+ -+ cur_size = kbase_mem_pool_size(pool); -+ -+ if (new_size > pool->max_size) -+ new_size = pool->max_size; -+ -+ if (new_size < cur_size) -+ kbase_mem_pool_shrink(pool, cur_size - new_size); -+ else if (new_size > cur_size) -+ kbase_mem_pool_grow(pool, new_size - cur_size); -+} -+ -+void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) -+{ -+ size_t cur_size; -+ size_t nr_to_shrink; -+ -+ kbase_mem_pool_lock(pool); -+ -+ pool->max_size = max_size; -+ -+ cur_size = kbase_mem_pool_size(pool); -+ if (max_size < cur_size) { -+ nr_to_shrink = cur_size - max_size; -+ kbase_mem_pool_shrink_locked(pool, nr_to_shrink); -+ } -+ -+ kbase_mem_pool_unlock(pool); -+} -+ -+ -+static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, -+ struct shrink_control *sc) -+{ -+ struct kbase_mem_pool *pool; -+ -+ pool = container_of(s, struct kbase_mem_pool, reclaim); -+ pool_dbg(pool, "reclaim count: %zu\n", kbase_mem_pool_size(pool)); -+ return kbase_mem_pool_size(pool); -+} -+ -+static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, -+ struct shrink_control *sc) -+{ -+ struct kbase_mem_pool *pool; -+ unsigned long freed; -+ -+ pool = container_of(s, struct kbase_mem_pool, reclaim); -+ -+ pool_dbg(pool, "reclaim scan %ld:\n", sc->nr_to_scan); -+ -+ freed = kbase_mem_pool_shrink(pool, sc->nr_to_scan); -+ -+ pool_dbg(pool, "reclaim freed %ld pages\n", freed); -+ -+ return freed; -+} -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, -+ struct shrink_control *sc) -+{ -+ if (sc->nr_to_scan == 0) -+ return kbase_mem_pool_reclaim_count_objects(s, sc); -+ -+ return kbase_mem_pool_reclaim_scan_objects(s, sc); -+} -+#endif -+ -+int kbase_mem_pool_init(struct kbase_mem_pool *pool, -+ size_t max_size, -+ struct kbase_device *kbdev, -+ struct kbase_mem_pool *next_pool) -+{ -+ pool->cur_size = 0; -+ pool->max_size = max_size; -+ pool->kbdev = kbdev; -+ pool->next_pool = next_pool; -+ -+ spin_lock_init(&pool->pool_lock); -+ INIT_LIST_HEAD(&pool->page_list); -+ -+ /* Register shrinker */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -+ pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; -+#else -+ pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; -+ pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; -+#endif -+ pool->reclaim.seeks = DEFAULT_SEEKS; -+ /* Kernel versions prior to 3.1 : -+ * struct shrinker does not define batch */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) -+ pool->reclaim.batch = 0; -+#endif -+ register_shrinker(&pool->reclaim); -+ -+ pool_dbg(pool, "initialized\n"); -+ -+ return 0; -+} -+ -+void kbase_mem_pool_term(struct kbase_mem_pool *pool) -+{ -+ struct kbase_mem_pool *next_pool = pool->next_pool; -+ struct page *p; -+ size_t nr_to_spill = 0; -+ LIST_HEAD(spill_list); -+ int i; -+ -+ pool_dbg(pool, "terminate()\n"); -+ -+ unregister_shrinker(&pool->reclaim); -+ -+ kbase_mem_pool_lock(pool); -+ pool->max_size = 0; -+ -+ if (next_pool && !kbase_mem_pool_is_full(next_pool)) { -+ /* Spill to next pool (may overspill) */ -+ nr_to_spill = kbase_mem_pool_capacity(next_pool); -+ nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); -+ -+ /* Zero pages first without holding the next_pool lock */ -+ for (i = 0; i < nr_to_spill; i++) { -+ p = kbase_mem_pool_remove_locked(pool); -+ kbase_mem_pool_zero_page(pool, p); -+ list_add(&p->lru, &spill_list); -+ } -+ } -+ -+ while (!kbase_mem_pool_is_empty(pool)) { -+ /* Free remaining pages to kernel */ -+ p = kbase_mem_pool_remove_locked(pool); -+ kbase_mem_pool_free_page(pool, p); -+ } -+ -+ kbase_mem_pool_unlock(pool); -+ -+ if (next_pool && nr_to_spill) { -+ /* Add new page list to next_pool */ -+ kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); -+ -+ pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); -+ } -+ -+ pool_dbg(pool, "terminated\n"); -+} -+ -+struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) -+{ -+ struct page *p; -+ -+ do { -+ pool_dbg(pool, "alloc()\n"); -+ p = kbase_mem_pool_remove(pool); -+ -+ if (p) -+ return p; -+ -+ pool = pool->next_pool; -+ } while (pool); -+ -+ return NULL; -+} -+ -+void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, -+ bool dirty) -+{ -+ struct kbase_mem_pool *next_pool = pool->next_pool; -+ -+ pool_dbg(pool, "free()\n"); -+ -+ if (!kbase_mem_pool_is_full(pool)) { -+ /* Add to our own pool */ -+ if (dirty) -+ kbase_mem_pool_sync_page(pool, p); -+ -+ kbase_mem_pool_add(pool, p); -+ } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { -+ /* Spill to next pool */ -+ kbase_mem_pool_spill(next_pool, p); -+ } else { -+ /* Free page */ -+ kbase_mem_pool_free_page(pool, p); -+ } -+} -+ -+int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages, -+ phys_addr_t *pages) -+{ -+ struct page *p; -+ size_t nr_from_pool; -+ size_t i; -+ int err = -ENOMEM; -+ -+ pool_dbg(pool, "alloc_pages(%zu):\n", nr_pages); -+ -+ /* Get pages from this pool */ -+ kbase_mem_pool_lock(pool); -+ nr_from_pool = min(nr_pages, kbase_mem_pool_size(pool)); -+ for (i = 0; i < nr_from_pool; i++) { -+ p = kbase_mem_pool_remove_locked(pool); -+ pages[i] = page_to_phys(p); -+ } -+ kbase_mem_pool_unlock(pool); -+ -+ if (i != nr_pages && pool->next_pool) { -+ /* Allocate via next pool */ -+ err = kbase_mem_pool_alloc_pages(pool->next_pool, -+ nr_pages - i, pages + i); -+ -+ if (err) -+ goto err_rollback; -+ -+ i += nr_pages - i; -+ } -+ -+ /* Get any remaining pages from kernel */ -+ for (; i < nr_pages; i++) { -+ p = kbase_mem_alloc_page(pool->kbdev); -+ if (!p) -+ goto err_rollback; -+ pages[i] = page_to_phys(p); -+ } -+ -+ pool_dbg(pool, "alloc_pages(%zu) done\n", nr_pages); -+ -+ return 0; -+ -+err_rollback: -+ kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); -+ return err; -+} -+ -+static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, -+ size_t nr_pages, phys_addr_t *pages, bool zero, bool sync) -+{ -+ struct page *p; -+ size_t nr_to_pool = 0; -+ LIST_HEAD(new_page_list); -+ size_t i; -+ -+ if (!nr_pages) -+ return; -+ -+ pool_dbg(pool, "add_array(%zu, zero=%d, sync=%d):\n", -+ nr_pages, zero, sync); -+ -+ /* Zero/sync pages first without holding the pool lock */ -+ for (i = 0; i < nr_pages; i++) { -+ if (unlikely(!pages[i])) -+ continue; -+ -+ p = phys_to_page(pages[i]); -+ -+ if (zero) -+ kbase_mem_pool_zero_page(pool, p); -+ else if (sync) -+ kbase_mem_pool_sync_page(pool, p); -+ -+ list_add(&p->lru, &new_page_list); -+ nr_to_pool++; -+ pages[i] = 0; -+ } -+ -+ /* Add new page list to pool */ -+ kbase_mem_pool_add_list(pool, &new_page_list, nr_to_pool); -+ -+ pool_dbg(pool, "add_array(%zu) added %zu pages\n", -+ nr_pages, nr_to_pool); -+} -+ -+void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, -+ phys_addr_t *pages, bool dirty, bool reclaimed) -+{ -+ struct kbase_mem_pool *next_pool = pool->next_pool; -+ struct page *p; -+ size_t nr_to_pool; -+ LIST_HEAD(to_pool_list); -+ size_t i = 0; -+ -+ pool_dbg(pool, "free_pages(%zu):\n", nr_pages); -+ -+ if (!reclaimed) { -+ /* Add to this pool */ -+ nr_to_pool = kbase_mem_pool_capacity(pool); -+ nr_to_pool = min(nr_pages, nr_to_pool); -+ -+ kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); -+ -+ i += nr_to_pool; -+ -+ if (i != nr_pages && next_pool) { -+ /* Spill to next pool (may overspill) */ -+ nr_to_pool = kbase_mem_pool_capacity(next_pool); -+ nr_to_pool = min(nr_pages - i, nr_to_pool); -+ -+ kbase_mem_pool_add_array(next_pool, nr_to_pool, -+ pages + i, true, dirty); -+ i += nr_to_pool; -+ } -+ } -+ -+ /* Free any remaining pages to kernel */ -+ for (; i < nr_pages; i++) { -+ if (unlikely(!pages[i])) -+ continue; -+ -+ p = phys_to_page(pages[i]); -+ -+ kbase_mem_pool_free_page(pool, p); -+ pages[i] = 0; -+ } -+ -+ pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c -new file mode 100755 -index 000000000..585fba036 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c -@@ -0,0 +1,81 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+ -+#include -+ -+#ifdef CONFIG_DEBUG_FS -+ -+static int kbase_mem_pool_debugfs_size_get(void *data, u64 *val) -+{ -+ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; -+ -+ *val = kbase_mem_pool_size(pool); -+ -+ return 0; -+} -+ -+static int kbase_mem_pool_debugfs_size_set(void *data, u64 val) -+{ -+ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; -+ -+ kbase_mem_pool_trim(pool, val); -+ -+ return 0; -+} -+ -+DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_size_fops, -+ kbase_mem_pool_debugfs_size_get, -+ kbase_mem_pool_debugfs_size_set, -+ "%llu\n"); -+ -+static int kbase_mem_pool_debugfs_max_size_get(void *data, u64 *val) -+{ -+ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; -+ -+ *val = kbase_mem_pool_max_size(pool); -+ -+ return 0; -+} -+ -+static int kbase_mem_pool_debugfs_max_size_set(void *data, u64 val) -+{ -+ struct kbase_mem_pool *pool = (struct kbase_mem_pool *)data; -+ -+ kbase_mem_pool_set_max_size(pool, val); -+ -+ return 0; -+} -+ -+DEFINE_SIMPLE_ATTRIBUTE(kbase_mem_pool_debugfs_max_size_fops, -+ kbase_mem_pool_debugfs_max_size_get, -+ kbase_mem_pool_debugfs_max_size_set, -+ "%llu\n"); -+ -+void kbase_mem_pool_debugfs_init(struct dentry *parent, -+ struct kbase_mem_pool *pool) -+{ -+ debugfs_create_file("mem_pool_size", S_IRUGO | S_IWUSR, parent, -+ pool, &kbase_mem_pool_debugfs_size_fops); -+ -+ debugfs_create_file("mem_pool_max_size", S_IRUGO | S_IWUSR, parent, -+ pool, &kbase_mem_pool_debugfs_max_size_fops); -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h -new file mode 100755 -index 000000000..1442854e8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h -@@ -0,0 +1,36 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_MEM_POOL_DEBUGFS_H -+#define _KBASE_MEM_POOL_DEBUGFS_H -+ -+#include -+ -+/** -+ * kbase_mem_pool_debugfs_init - add debugfs knobs for @pool -+ * @parent: Parent debugfs dentry -+ * @pool: Memory pool to control -+ * -+ * Adds two debugfs files under @parent: -+ * - mem_pool_size: get/set the current size of @pool -+ * - mem_pool_max_size: get/set the max size of @pool -+ */ -+void kbase_mem_pool_debugfs_init(struct dentry *parent, -+ struct kbase_mem_pool *pool); -+ -+#endif /*_KBASE_MEM_POOL_DEBUGFS_H*/ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c -new file mode 100755 -index 000000000..d58fd8d62 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c -@@ -0,0 +1,121 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+ -+#ifdef CONFIG_DEBUG_FS -+ -+/** Show callback for the @c mem_profile debugfs file. -+ * -+ * This function is called to get the contents of the @c mem_profile debugfs -+ * file. This is a report of current memory usage and distribution in userspace. -+ * -+ * @param sfile The debugfs entry -+ * @param data Data associated with the entry -+ * -+ * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise -+ */ -+static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) -+{ -+ struct kbase_context *kctx = sfile->private; -+ -+ mutex_lock(&kctx->mem_profile_lock); -+ -+ seq_write(sfile, kctx->mem_profile_data, kctx->mem_profile_size); -+ -+ seq_putc(sfile, '\n'); -+ -+ mutex_unlock(&kctx->mem_profile_lock); -+ -+ return 0; -+} -+ -+/* -+ * File operations related to debugfs entry for mem_profile -+ */ -+static int kbasep_mem_profile_debugfs_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbasep_mem_profile_seq_show, in->i_private); -+} -+ -+static const struct file_operations kbasep_mem_profile_debugfs_fops = { -+ .open = kbasep_mem_profile_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, -+ size_t size) -+{ -+ int err = 0; -+ -+ mutex_lock(&kctx->mem_profile_lock); -+ -+ dev_dbg(kctx->kbdev->dev, "initialised: %d", -+ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); -+ -+ if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { -+ if (!debugfs_create_file("mem_profile", S_IRUGO, -+ kctx->kctx_dentry, kctx, -+ &kbasep_mem_profile_debugfs_fops)) { -+ err = -EAGAIN; -+ } else { -+ kbase_ctx_flag_set(kctx, -+ KCTX_MEM_PROFILE_INITIALIZED); -+ } -+ } -+ -+ if (kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { -+ kfree(kctx->mem_profile_data); -+ kctx->mem_profile_data = data; -+ kctx->mem_profile_size = size; -+ } else { -+ kfree(data); -+ } -+ -+ dev_dbg(kctx->kbdev->dev, "returning: %d, initialised: %d", -+ err, kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); -+ -+ mutex_unlock(&kctx->mem_profile_lock); -+ -+ return err; -+} -+ -+void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx) -+{ -+ mutex_lock(&kctx->mem_profile_lock); -+ -+ dev_dbg(kctx->kbdev->dev, "initialised: %d", -+ kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)); -+ -+ kfree(kctx->mem_profile_data); -+ kctx->mem_profile_data = NULL; -+ kctx->mem_profile_size = 0; -+ -+ mutex_unlock(&kctx->mem_profile_lock); -+} -+ -+#else /* CONFIG_DEBUG_FS */ -+ -+int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, -+ size_t size) -+{ -+ kfree(data); -+ return 0; -+} -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h -new file mode 100755 -index 000000000..a1dc2e0b1 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h -@@ -0,0 +1,59 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_mem_profile_debugfs.h -+ * Header file for mem profiles entries in debugfs -+ * -+ */ -+ -+#ifndef _KBASE_MEM_PROFILE_DEBUGFS_H -+#define _KBASE_MEM_PROFILE_DEBUGFS_H -+ -+#include -+#include -+ -+/** -+ * @brief Remove entry from Mali memory profile debugfs -+ */ -+void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); -+ -+/** -+ * @brief Insert @p data to the debugfs file so it can be read by userspace -+ * -+ * The function takes ownership of @p data and frees it later when new data -+ * is inserted. -+ * -+ * If the debugfs entry corresponding to the @p kctx doesn't exist, -+ * an attempt will be made to create it. -+ * -+ * @param kctx The context whose debugfs file @p data should be inserted to -+ * @param data A NULL-terminated string to be inserted to the debugfs file, -+ * without the trailing new line character -+ * @param size The length of the @p data string -+ * @return 0 if @p data inserted correctly -+ * -EAGAIN in case of error -+ * @post @ref mem_profile_initialized will be set to @c true -+ * the first time this function succeeds. -+ */ -+int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, -+ size_t size); -+ -+#endif /*_KBASE_MEM_PROFILE_DEBUGFS_H*/ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h -new file mode 100755 -index 000000000..82f070297 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h -@@ -0,0 +1,33 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file mali_kbase_mem_profile_debugfs_buf_size.h -+ * Header file for the size of the buffer to accumulate the histogram report text in -+ */ -+ -+#ifndef _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ -+#define _KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_ -+ -+/** -+ * The size of the buffer to accumulate the histogram report text in -+ * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT -+ */ -+#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t) (64 + ((80 + (56 * 64)) * 15) + 56)) -+ -+#endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu.c b/drivers/gpu/arm/midgard/mali_kbase_mmu.c -new file mode 100755 -index 000000000..26144850a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mmu.c -@@ -0,0 +1,2088 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_mmu.c -+ * Base kernel MMU management. -+ */ -+ -+/* #define DEBUG 1 */ -+#include -+#include -+#include -+#include -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+#include -+#endif -+#include -+#include -+#include -+ -+#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define KBASE_MMU_PAGE_ENTRIES 512 -+ -+/** -+ * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. -+ * @kctx: The KBase context. -+ * @vpfn: The virtual page frame number to start the flush on. -+ * @nr: The number of pages to flush. -+ * @sync: Set if the operation should be synchronous or not. -+ * -+ * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. -+ * -+ * If sync is not set then transactions still in flight when the flush is issued -+ * may use the old page tables and the data they write will not be written out -+ * to memory, this function returns after the flush has been issued but -+ * before all accesses which might effect the flushed region have completed. -+ * -+ * If sync is set then accesses in the flushed region will be drained -+ * before data is flush and invalidated through L1, L2 and into memory, -+ * after which point this function will return. -+ */ -+static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, -+ u64 vpfn, size_t nr, bool sync); -+ -+/** -+ * kbase_mmu_sync_pgd - sync page directory to memory -+ * @kbdev: Device pointer. -+ * @handle: Address of DMA region. -+ * @size: Size of the region to sync. -+ * -+ * This should be called after each page directory update. -+ */ -+ -+static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, -+ dma_addr_t handle, size_t size) -+{ -+ /* If page table is not coherent then ensure the gpu can read -+ * the pages from memory -+ */ -+ if (kbdev->system_coherency != COHERENCY_ACE) -+ dma_sync_single_for_device(kbdev->dev, handle, size, -+ DMA_TO_DEVICE); -+} -+ -+/* -+ * Definitions: -+ * - PGD: Page Directory. -+ * - PTE: Page Table Entry. A 64bit value pointing to the next -+ * level of translation -+ * - ATE: Address Transation Entry. A 64bit value pointing to -+ * a 4kB physical page. -+ */ -+ -+static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, -+ struct kbase_as *as, const char *reason_str); -+ -+ -+static size_t make_multiple(size_t minimum, size_t multiple) -+{ -+ size_t remainder = minimum % multiple; -+ -+ if (remainder == 0) -+ return minimum; -+ -+ return minimum + multiple - remainder; -+} -+ -+void page_fault_worker(struct work_struct *data) -+{ -+ u64 fault_pfn; -+ u32 fault_status; -+ size_t new_pages; -+ size_t fault_rel_pfn; -+ struct kbase_as *faulting_as; -+ int as_no; -+ struct kbase_context *kctx; -+ struct kbase_device *kbdev; -+ struct kbase_va_region *region; -+ int err; -+ bool grown = false; -+ -+ faulting_as = container_of(data, struct kbase_as, work_pagefault); -+ fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; -+ as_no = faulting_as->number; -+ -+ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); -+ -+ /* Grab the context that was already refcounted in kbase_mmu_interrupt(). -+ * Therefore, it cannot be scheduled out of this AS until we explicitly release it -+ */ -+ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); -+ if (WARN_ON(!kctx)) { -+ atomic_dec(&kbdev->faults_pending); -+ return; -+ } -+ -+ KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); -+ -+ if (unlikely(faulting_as->protected_mode)) -+ { -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Protected mode fault"); -+ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ -+ goto fault_done; -+ } -+ -+ fault_status = faulting_as->fault_status; -+ switch (fault_status & AS_FAULTSTATUS_EXCEPTION_CODE_MASK) { -+ -+ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT: -+ /* need to check against the region to handle this one */ -+ break; -+ -+ case AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT: -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Permission failure"); -+ goto fault_done; -+ -+ case AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT: -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Translation table bus fault"); -+ goto fault_done; -+ -+ case AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG: -+ /* nothing to do, but we don't expect this fault currently */ -+ dev_warn(kbdev->dev, "Access flag unexpectedly set"); -+ goto fault_done; -+ -+ case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Address size fault"); -+ else -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Unknown fault code"); -+ goto fault_done; -+ -+ case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Memory attributes fault"); -+ else -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Unknown fault code"); -+ goto fault_done; -+ -+ default: -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Unknown fault code"); -+ goto fault_done; -+ } -+ -+ /* so we have a translation fault, let's see if it is for growable -+ * memory */ -+ kbase_gpu_vm_lock(kctx); -+ -+ region = kbase_region_tracker_find_region_enclosing_address(kctx, -+ faulting_as->fault_addr); -+ if (!region || region->flags & KBASE_REG_FREE) { -+ kbase_gpu_vm_unlock(kctx); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Memory is not mapped on the GPU"); -+ goto fault_done; -+ } -+ -+ if (region->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { -+ kbase_gpu_vm_unlock(kctx); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "DMA-BUF is not mapped on the GPU"); -+ goto fault_done; -+ } -+ -+ if ((region->flags & GROWABLE_FLAGS_REQUIRED) -+ != GROWABLE_FLAGS_REQUIRED) { -+ kbase_gpu_vm_unlock(kctx); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Memory is not growable"); -+ goto fault_done; -+ } -+ -+ if ((region->flags & KBASE_REG_DONT_NEED)) { -+ kbase_gpu_vm_unlock(kctx); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Don't need memory can't be grown"); -+ goto fault_done; -+ } -+ -+ /* find the size we need to grow it by */ -+ /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address -+ * validating the fault_adress to be within a size_t from the start_pfn */ -+ fault_rel_pfn = fault_pfn - region->start_pfn; -+ -+ if (fault_rel_pfn < kbase_reg_current_backed_size(region)) { -+ dev_dbg(kbdev->dev, "Page fault @ 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", -+ faulting_as->fault_addr, region->start_pfn, -+ region->start_pfn + -+ kbase_reg_current_backed_size(region)); -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ /* [1] in case another page fault occurred while we were -+ * handling the (duplicate) page fault we need to ensure we -+ * don't loose the other page fault as result of us clearing -+ * the MMU IRQ. Therefore, after we clear the MMU IRQ we send -+ * an UNLOCK command that will retry any stalled memory -+ * transaction (which should cause the other page fault to be -+ * raised again). -+ */ -+ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, -+ AS_COMMAND_UNLOCK, 1); -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ kbase_gpu_vm_unlock(kctx); -+ -+ goto fault_done; -+ } -+ -+ new_pages = make_multiple(fault_rel_pfn - -+ kbase_reg_current_backed_size(region) + 1, -+ region->extent); -+ -+ /* cap to max vsize */ -+ if (new_pages + kbase_reg_current_backed_size(region) > -+ region->nr_pages) -+ new_pages = region->nr_pages - -+ kbase_reg_current_backed_size(region); -+ -+ if (0 == new_pages) { -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ /* Duplicate of a fault we've already handled, nothing to do */ -+ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ /* See comment [1] about UNLOCK usage */ -+ kbase_mmu_hw_do_operation(kbdev, faulting_as, NULL, 0, 0, -+ AS_COMMAND_UNLOCK, 1); -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ kbase_gpu_vm_unlock(kctx); -+ goto fault_done; -+ } -+ -+ if (kbase_alloc_phy_pages_helper(region->gpu_alloc, new_pages) == 0) { -+ if (region->gpu_alloc != region->cpu_alloc) { -+ if (kbase_alloc_phy_pages_helper( -+ region->cpu_alloc, new_pages) == 0) { -+ grown = true; -+ } else { -+ kbase_free_phy_pages_helper(region->gpu_alloc, -+ new_pages); -+ } -+ } else { -+ grown = true; -+ } -+ } -+ -+ -+ if (grown) { -+ u64 pfn_offset; -+ u32 op; -+ -+ /* alloc success */ -+ KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages); -+ -+ /* set up the new pages */ -+ pfn_offset = kbase_reg_current_backed_size(region) - new_pages; -+ /* -+ * Note: -+ * Issuing an MMU operation will unlock the MMU and cause the -+ * translation to be replayed. If the page insertion fails then -+ * rather then trying to continue the context should be killed -+ * so the no_flush version of insert_pages is used which allows -+ * us to unlock the MMU as we see fit. -+ */ -+ err = kbase_mmu_insert_pages_no_flush(kctx, -+ region->start_pfn + pfn_offset, -+ &kbase_get_gpu_phy_pages(region)[pfn_offset], -+ new_pages, region->flags); -+ if (err) { -+ kbase_free_phy_pages_helper(region->gpu_alloc, new_pages); -+ if (region->gpu_alloc != region->cpu_alloc) -+ kbase_free_phy_pages_helper(region->cpu_alloc, -+ new_pages); -+ kbase_gpu_vm_unlock(kctx); -+ /* The locked VA region will be unlocked and the cache invalidated in here */ -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Page table update failure"); -+ goto fault_done; -+ } -+#if defined(CONFIG_MALI_GATOR_SUPPORT) -+ kbase_trace_mali_page_fault_insert_pages(as_no, new_pages); -+#endif -+ KBASE_TLSTREAM_AUX_PAGEFAULT(kctx->id, (u64)new_pages); -+ -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ /* flush L2 and unlock the VA (resumes the MMU) */ -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) -+ op = AS_COMMAND_FLUSH; -+ else -+ op = AS_COMMAND_FLUSH_PT; -+ -+ /* clear MMU interrupt - this needs to be done after updating -+ * the page tables but before issuing a FLUSH command. The -+ * FLUSH cmd has a side effect that it restarts stalled memory -+ * transactions in other address spaces which may cause -+ * another fault to occur. If we didn't clear the interrupt at -+ * this stage a new IRQ might not be raised when the GPU finds -+ * a MMU IRQ is already pending. -+ */ -+ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ -+ kbase_mmu_hw_do_operation(kbdev, faulting_as, kctx, -+ faulting_as->fault_addr >> PAGE_SHIFT, -+ new_pages, -+ op, 1); -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ /* AS transaction end */ -+ -+ /* reenable this in the mask */ -+ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE); -+ kbase_gpu_vm_unlock(kctx); -+ } else { -+ /* failed to extend, handle as a normal PF */ -+ kbase_gpu_vm_unlock(kctx); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Page allocation failure"); -+ } -+ -+fault_done: -+ /* -+ * By this point, the fault was handled in some way, -+ * so release the ctx refcount -+ */ -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ -+ atomic_dec(&kbdev->faults_pending); -+} -+ -+phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx) -+{ -+ u64 *page; -+ int i; -+ struct page *p; -+ int new_page_count __maybe_unused; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ new_page_count = kbase_atomic_add_pages(1, &kctx->used_pages); -+ kbase_atomic_add_pages(1, &kctx->kbdev->memdev.used_pages); -+ -+ p = kbase_mem_pool_alloc(&kctx->mem_pool); -+ if (!p) -+ goto sub_pages; -+ -+ KBASE_TLSTREAM_AUX_PAGESALLOC( -+ (u32)kctx->id, -+ (u64)new_page_count); -+ -+ page = kmap(p); -+ if (NULL == page) -+ goto alloc_free; -+ -+ kbase_process_page_usage_inc(kctx, 1); -+ -+ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) -+ kctx->kbdev->mmu_mode->entry_invalidate(&page[i]); -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); -+ -+ kunmap(p); -+ return page_to_phys(p); -+ -+alloc_free: -+ kbase_mem_pool_free(&kctx->mem_pool, p, false); -+sub_pages: -+ kbase_atomic_sub_pages(1, &kctx->used_pages); -+ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mmu_alloc_pgd); -+ -+/* Given PGD PFN for level N, return PGD PFN for level N+1, allocating the -+ * new table from the pool if needed and possible -+ */ -+static int mmu_get_next_pgd(struct kbase_context *kctx, -+ phys_addr_t *pgd, u64 vpfn, int level) -+{ -+ u64 *page; -+ phys_addr_t target_pgd; -+ struct page *p; -+ -+ KBASE_DEBUG_ASSERT(*pgd); -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ lockdep_assert_held(&kctx->mmu_lock); -+ -+ /* -+ * Architecture spec defines level-0 as being the top-most. -+ * This is a bit unfortunate here, but we keep the same convention. -+ */ -+ vpfn >>= (3 - level) * 9; -+ vpfn &= 0x1FF; -+ -+ p = pfn_to_page(PFN_DOWN(*pgd)); -+ page = kmap(p); -+ if (NULL == page) { -+ dev_warn(kctx->kbdev->dev, "mmu_get_next_pgd: kmap failure\n"); -+ return -EINVAL; -+ } -+ -+ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); -+ -+ if (!target_pgd) { -+ target_pgd = kbase_mmu_alloc_pgd(kctx); -+ if (!target_pgd) { -+ dev_dbg(kctx->kbdev->dev, "mmu_get_next_pgd: kbase_mmu_alloc_pgd failure\n"); -+ kunmap(p); -+ return -ENOMEM; -+ } -+ -+ kctx->kbdev->mmu_mode->entry_set_pte(&page[vpfn], target_pgd); -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); -+ /* Rely on the caller to update the address space flags. */ -+ } -+ -+ kunmap(p); -+ *pgd = target_pgd; -+ -+ return 0; -+} -+ -+static int mmu_get_bottom_pgd(struct kbase_context *kctx, -+ u64 vpfn, phys_addr_t *out_pgd) -+{ -+ phys_addr_t pgd; -+ int l; -+ -+ lockdep_assert_held(&kctx->mmu_lock); -+ -+ pgd = kctx->pgd; -+ for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { -+ int err = mmu_get_next_pgd(kctx, &pgd, vpfn, l); -+ /* Handle failure condition */ -+ if (err) { -+ dev_dbg(kctx->kbdev->dev, "mmu_get_bottom_pgd: mmu_get_next_pgd failure\n"); -+ return err; -+ } -+ } -+ -+ *out_pgd = pgd; -+ -+ return 0; -+} -+ -+static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, u64 vpfn, int level) -+{ -+ u64 *page; -+ phys_addr_t target_pgd; -+ -+ KBASE_DEBUG_ASSERT(pgd); -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ -+ lockdep_assert_held(&kctx->mmu_lock); -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ /* -+ * Architecture spec defines level-0 as being the top-most. -+ * This is a bit unfortunate here, but we keep the same convention. -+ */ -+ vpfn >>= (3 - level) * 9; -+ vpfn &= 0x1FF; -+ -+ page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); -+ /* kmap_atomic should NEVER fail */ -+ KBASE_DEBUG_ASSERT(NULL != page); -+ -+ target_pgd = kctx->kbdev->mmu_mode->pte_to_phy_addr(page[vpfn]); -+ /* As we are recovering from what has already been set up, we should have a target_pgd */ -+ KBASE_DEBUG_ASSERT(0 != target_pgd); -+ kunmap_atomic(page); -+ return target_pgd; -+} -+ -+static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn) -+{ -+ phys_addr_t pgd; -+ int l; -+ -+ lockdep_assert_held(&kctx->mmu_lock); -+ -+ pgd = kctx->pgd; -+ -+ for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { -+ pgd = mmu_insert_pages_recover_get_next_pgd(kctx, pgd, vpfn, l); -+ /* Should never fail */ -+ KBASE_DEBUG_ASSERT(0 != pgd); -+ } -+ -+ return pgd; -+} -+ -+static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vpfn, -+ size_t nr) -+{ -+ phys_addr_t pgd; -+ u64 *pgd_page; -+ struct kbase_mmu_mode const *mmu_mode; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(0 != vpfn); -+ /* 64-bit address range is the max */ -+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); -+ -+ lockdep_assert_held(&kctx->mmu_lock); -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ mmu_mode = kctx->kbdev->mmu_mode; -+ -+ while (nr) { -+ unsigned int i; -+ unsigned int index = vpfn & 0x1FF; -+ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; -+ struct page *p; -+ -+ if (count > nr) -+ count = nr; -+ -+ pgd = mmu_insert_pages_recover_get_bottom_pgd(kctx, vpfn); -+ KBASE_DEBUG_ASSERT(0 != pgd); -+ -+ p = pfn_to_page(PFN_DOWN(pgd)); -+ -+ pgd_page = kmap_atomic(p); -+ KBASE_DEBUG_ASSERT(NULL != pgd_page); -+ -+ /* Invalidate the entries we added */ -+ for (i = 0; i < count; i++) -+ mmu_mode->entry_invalidate(&pgd_page[index + i]); -+ -+ vpfn += count; -+ nr -= count; -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p), PAGE_SIZE); -+ -+ kunmap_atomic(pgd_page); -+ } -+} -+ -+/* -+ * Map the single page 'phys' 'nr' of times, starting at GPU PFN 'vpfn' -+ */ -+int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, -+ phys_addr_t phys, size_t nr, -+ unsigned long flags) -+{ -+ phys_addr_t pgd; -+ u64 *pgd_page; -+ /* In case the insert_single_page only partially completes we need to be -+ * able to recover */ -+ bool recover_required = false; -+ u64 recover_vpfn = vpfn; -+ size_t recover_count = 0; -+ size_t remain = nr; -+ int err; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(0 != vpfn); -+ /* 64-bit address range is the max */ -+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); -+ -+ /* Early out if there is nothing to do */ -+ if (nr == 0) -+ return 0; -+ -+ mutex_lock(&kctx->mmu_lock); -+ -+ while (remain) { -+ unsigned int i; -+ unsigned int index = vpfn & 0x1FF; -+ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; -+ struct page *p; -+ -+ if (count > remain) -+ count = remain; -+ -+ /* -+ * Repeatedly calling mmu_get_bottom_pte() is clearly -+ * suboptimal. We don't have to re-parse the whole tree -+ * each time (just cache the l0-l2 sequence). -+ * On the other hand, it's only a gain when we map more than -+ * 256 pages at once (on average). Do we really care? -+ */ -+ do { -+ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); -+ if (err != -ENOMEM) -+ break; -+ /* Fill the memory pool with enough pages for -+ * the page walk to succeed -+ */ -+ mutex_unlock(&kctx->mmu_lock); -+ err = kbase_mem_pool_grow(&kctx->mem_pool, -+ MIDGARD_MMU_BOTTOMLEVEL); -+ mutex_lock(&kctx->mmu_lock); -+ } while (!err); -+ if (err) { -+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); -+ if (recover_required) { -+ /* Invalidate the pages we have partially -+ * completed */ -+ mmu_insert_pages_failure_recovery(kctx, -+ recover_vpfn, -+ recover_count); -+ } -+ goto fail_unlock; -+ } -+ -+ p = pfn_to_page(PFN_DOWN(pgd)); -+ pgd_page = kmap(p); -+ if (!pgd_page) { -+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); -+ if (recover_required) { -+ /* Invalidate the pages we have partially -+ * completed */ -+ mmu_insert_pages_failure_recovery(kctx, -+ recover_vpfn, -+ recover_count); -+ } -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ for (i = 0; i < count; i++) { -+ unsigned int ofs = index + i; -+ -+ KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); -+ kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs], -+ phys, flags); -+ } -+ -+ vpfn += count; -+ remain -= count; -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, -+ kbase_dma_addr(p) + (index * sizeof(u64)), -+ count * sizeof(u64)); -+ -+ kunmap(p); -+ /* We have started modifying the page table. -+ * If further pages need inserting and fail we need to undo what -+ * has already taken place */ -+ recover_required = true; -+ recover_count += count; -+ } -+ mutex_unlock(&kctx->mmu_lock); -+ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); -+ return 0; -+ -+fail_unlock: -+ mutex_unlock(&kctx->mmu_lock); -+ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); -+ return err; -+} -+ -+int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, -+ phys_addr_t *phys, size_t nr, -+ unsigned long flags) -+{ -+ phys_addr_t pgd; -+ u64 *pgd_page; -+ /* In case the insert_pages only partially completes we need to be able -+ * to recover */ -+ bool recover_required = false; -+ u64 recover_vpfn = vpfn; -+ size_t recover_count = 0; -+ size_t remain = nr; -+ int err; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(0 != vpfn); -+ /* 64-bit address range is the max */ -+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); -+ -+ /* Early out if there is nothing to do */ -+ if (nr == 0) -+ return 0; -+ -+ mutex_lock(&kctx->mmu_lock); -+ -+ while (remain) { -+ unsigned int i; -+ unsigned int index = vpfn & 0x1FF; -+ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; -+ struct page *p; -+ -+ if (count > remain) -+ count = remain; -+ -+ /* -+ * Repeatedly calling mmu_get_bottom_pte() is clearly -+ * suboptimal. We don't have to re-parse the whole tree -+ * each time (just cache the l0-l2 sequence). -+ * On the other hand, it's only a gain when we map more than -+ * 256 pages at once (on average). Do we really care? -+ */ -+ do { -+ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); -+ if (err != -ENOMEM) -+ break; -+ /* Fill the memory pool with enough pages for -+ * the page walk to succeed -+ */ -+ mutex_unlock(&kctx->mmu_lock); -+ err = kbase_mem_pool_grow(&kctx->mem_pool, -+ MIDGARD_MMU_BOTTOMLEVEL); -+ mutex_lock(&kctx->mmu_lock); -+ } while (!err); -+ if (err) { -+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: mmu_get_bottom_pgd failure\n"); -+ if (recover_required) { -+ /* Invalidate the pages we have partially -+ * completed */ -+ mmu_insert_pages_failure_recovery(kctx, -+ recover_vpfn, -+ recover_count); -+ } -+ goto fail_unlock; -+ } -+ -+ p = pfn_to_page(PFN_DOWN(pgd)); -+ pgd_page = kmap(p); -+ if (!pgd_page) { -+ dev_warn(kctx->kbdev->dev, "kbase_mmu_insert_pages: kmap failure\n"); -+ if (recover_required) { -+ /* Invalidate the pages we have partially -+ * completed */ -+ mmu_insert_pages_failure_recovery(kctx, -+ recover_vpfn, -+ recover_count); -+ } -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ for (i = 0; i < count; i++) { -+ unsigned int ofs = index + i; -+ -+ KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); -+ kctx->kbdev->mmu_mode->entry_set_ate(&pgd_page[ofs], -+ phys[i], flags); -+ } -+ -+ phys += count; -+ vpfn += count; -+ remain -= count; -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, -+ kbase_dma_addr(p) + (index * sizeof(u64)), -+ count * sizeof(u64)); -+ -+ kunmap(p); -+ /* We have started modifying the page table. If further pages -+ * need inserting and fail we need to undo what has already -+ * taken place */ -+ recover_required = true; -+ recover_count += count; -+ } -+ -+ mutex_unlock(&kctx->mmu_lock); -+ return 0; -+ -+fail_unlock: -+ mutex_unlock(&kctx->mmu_lock); -+ return err; -+} -+ -+/* -+ * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' -+ */ -+int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, -+ phys_addr_t *phys, size_t nr, -+ unsigned long flags) -+{ -+ int err; -+ -+ err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags); -+ kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); -+ -+/** -+ * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches -+ * without retaining the kbase context. -+ * @kctx: The KBase context. -+ * @vpfn: The virtual page frame number to start the flush on. -+ * @nr: The number of pages to flush. -+ * @sync: Set if the operation should be synchronous or not. -+ * -+ * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any -+ * other locking. -+ */ -+static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, -+ u64 vpfn, size_t nr, bool sync) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ int err; -+ u32 op; -+ -+ /* Early out if there is nothing to do */ -+ if (nr == 0) -+ return; -+ -+ if (sync) -+ op = AS_COMMAND_FLUSH_MEM; -+ else -+ op = AS_COMMAND_FLUSH_PT; -+ -+ err = kbase_mmu_hw_do_operation(kbdev, -+ &kbdev->as[kctx->as_nr], -+ kctx, vpfn, nr, op, 0); -+#if KBASE_GPU_RESET_EN -+ if (err) { -+ /* Flush failed to complete, assume the -+ * GPU has hung and perform a reset to -+ * recover */ -+ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); -+ -+ if (kbase_prepare_to_reset_gpu_locked(kbdev)) -+ kbase_reset_gpu_locked(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ -+#ifndef CONFIG_MALI_NO_MALI -+ /* -+ * As this function could be called in interrupt context the sync -+ * request can't block. Instead log the request and the next flush -+ * request will pick it up. -+ */ -+ if ((!err) && sync && -+ kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) -+ atomic_set(&kctx->drain_pending, 1); -+#endif /* !CONFIG_MALI_NO_MALI */ -+} -+ -+static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, -+ u64 vpfn, size_t nr, bool sync) -+{ -+ struct kbase_device *kbdev; -+ bool ctx_is_in_runpool; -+#ifndef CONFIG_MALI_NO_MALI -+ bool drain_pending = false; -+ -+ if (atomic_xchg(&kctx->drain_pending, 0)) -+ drain_pending = true; -+#endif /* !CONFIG_MALI_NO_MALI */ -+ -+ /* Early out if there is nothing to do */ -+ if (nr == 0) -+ return; -+ -+ kbdev = kctx->kbdev; -+ mutex_lock(&kbdev->js_data.queue_mutex); -+ ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); -+ mutex_unlock(&kbdev->js_data.queue_mutex); -+ -+ if (ctx_is_in_runpool) { -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ -+ if (!kbase_pm_context_active_handle_suspend(kbdev, -+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { -+ int err; -+ u32 op; -+ -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ if (sync) -+ op = AS_COMMAND_FLUSH_MEM; -+ else -+ op = AS_COMMAND_FLUSH_PT; -+ -+ err = kbase_mmu_hw_do_operation(kbdev, -+ &kbdev->as[kctx->as_nr], -+ kctx, vpfn, nr, op, 0); -+ -+#if KBASE_GPU_RESET_EN -+ if (err) { -+ /* Flush failed to complete, assume the -+ * GPU has hung and perform a reset to -+ * recover */ -+ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); -+ -+ if (kbase_prepare_to_reset_gpu(kbdev)) -+ kbase_reset_gpu(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ /* AS transaction end */ -+ -+#ifndef CONFIG_MALI_NO_MALI -+ /* -+ * The transaction lock must be dropped before here -+ * as kbase_wait_write_flush could take it if -+ * the GPU was powered down (static analysis doesn't -+ * know this can't happen). -+ */ -+ drain_pending |= (!err) && sync && -+ kbase_hw_has_issue(kctx->kbdev, -+ BASE_HW_ISSUE_6367); -+ if (drain_pending) { -+ /* Wait for GPU to flush write buffer */ -+ kbase_wait_write_flush(kctx); -+ } -+#endif /* !CONFIG_MALI_NO_MALI */ -+ -+ kbase_pm_context_idle(kbdev); -+ } -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ } -+} -+ -+void kbase_mmu_update(struct kbase_context *kctx) -+{ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); -+ /* ASSERT that the context has a valid as_nr, which is only the case -+ * when it's scheduled in. -+ * -+ * as_nr won't change because the caller has the hwaccess_lock */ -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ -+ kctx->kbdev->mmu_mode->update(kctx); -+} -+KBASE_EXPORT_TEST_API(kbase_mmu_update); -+ -+void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ lockdep_assert_held(&kbdev->mmu_hw_mutex); -+ -+ kbdev->mmu_mode->disable_as(kbdev, as_nr); -+} -+ -+void kbase_mmu_disable(struct kbase_context *kctx) -+{ -+ /* ASSERT that the context has a valid as_nr, which is only the case -+ * when it's scheduled in. -+ * -+ * as_nr won't change because the caller has the hwaccess_lock */ -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ -+ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ -+ /* -+ * The address space is being disabled, drain all knowledge of it out -+ * from the caches as pages and page tables might be freed after this. -+ * -+ * The job scheduler code will already be holding the locks and context -+ * so just do the flush. -+ */ -+ kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); -+ -+ kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); -+} -+KBASE_EXPORT_TEST_API(kbase_mmu_disable); -+ -+/* -+ * We actually only discard the ATE, and not the page table -+ * pages. There is a potential DoS here, as we'll leak memory by -+ * having PTEs that are potentially unused. Will require physical -+ * page accounting, so MMU pages are part of the process allocation. -+ * -+ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is -+ * currently scheduled into the runpool, and so potentially uses a lot of locks. -+ * These locks must be taken in the correct order with respect to others -+ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more -+ * information. -+ */ -+int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) -+{ -+ phys_addr_t pgd; -+ u64 *pgd_page; -+ struct kbase_device *kbdev; -+ size_t requested_nr = nr; -+ struct kbase_mmu_mode const *mmu_mode; -+ int err; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr); -+ -+ if (0 == nr) { -+ /* early out if nothing to do */ -+ return 0; -+ } -+ -+ mutex_lock(&kctx->mmu_lock); -+ -+ kbdev = kctx->kbdev; -+ mmu_mode = kbdev->mmu_mode; -+ -+ while (nr) { -+ unsigned int i; -+ unsigned int index = vpfn & 0x1FF; -+ unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; -+ struct page *p; -+ -+ if (count > nr) -+ count = nr; -+ -+ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); -+ if (err) { -+ dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n"); -+ err = -EINVAL; -+ goto fail_unlock; -+ } -+ -+ p = pfn_to_page(PFN_DOWN(pgd)); -+ pgd_page = kmap(p); -+ if (!pgd_page) { -+ dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n"); -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ for (i = 0; i < count; i++) -+ mmu_mode->entry_invalidate(&pgd_page[index + i]); -+ -+ vpfn += count; -+ nr -= count; -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, -+ kbase_dma_addr(p) + (index * sizeof(u64)), -+ count * sizeof(u64)); -+ -+ kunmap(p); -+ } -+ -+ mutex_unlock(&kctx->mmu_lock); -+ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); -+ return 0; -+ -+fail_unlock: -+ mutex_unlock(&kctx->mmu_lock); -+ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); -+ -+/** -+ * Update the entries for specified number of pages pointed to by 'phys' at GPU PFN 'vpfn'. -+ * This call is being triggered as a response to the changes of the mem attributes -+ * -+ * @pre : The caller is responsible for validating the memory attributes -+ * -+ * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is -+ * currently scheduled into the runpool, and so potentially uses a lot of locks. -+ * These locks must be taken in the correct order with respect to others -+ * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more -+ * information. -+ */ -+int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags) -+{ -+ phys_addr_t pgd; -+ u64 *pgd_page; -+ size_t requested_nr = nr; -+ struct kbase_mmu_mode const *mmu_mode; -+ int err; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(0 != vpfn); -+ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); -+ -+ /* Early out if there is nothing to do */ -+ if (nr == 0) -+ return 0; -+ -+ mutex_lock(&kctx->mmu_lock); -+ -+ mmu_mode = kctx->kbdev->mmu_mode; -+ -+ dev_warn(kctx->kbdev->dev, "kbase_mmu_update_pages(): updating page share flags on GPU PFN 0x%llx from phys %p, %zu pages", -+ vpfn, phys, nr); -+ -+ while (nr) { -+ unsigned int i; -+ unsigned int index = vpfn & 0x1FF; -+ size_t count = KBASE_MMU_PAGE_ENTRIES - index; -+ struct page *p; -+ -+ if (count > nr) -+ count = nr; -+ -+ do { -+ err = mmu_get_bottom_pgd(kctx, vpfn, &pgd); -+ if (err != -ENOMEM) -+ break; -+ /* Fill the memory pool with enough pages for -+ * the page walk to succeed -+ */ -+ mutex_unlock(&kctx->mmu_lock); -+ err = kbase_mem_pool_grow(&kctx->mem_pool, -+ MIDGARD_MMU_BOTTOMLEVEL); -+ mutex_lock(&kctx->mmu_lock); -+ } while (!err); -+ if (err) { -+ dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n"); -+ goto fail_unlock; -+ } -+ -+ p = pfn_to_page(PFN_DOWN(pgd)); -+ pgd_page = kmap(p); -+ if (!pgd_page) { -+ dev_warn(kctx->kbdev->dev, "kmap failure\n"); -+ err = -ENOMEM; -+ goto fail_unlock; -+ } -+ -+ for (i = 0; i < count; i++) -+ mmu_mode->entry_set_ate(&pgd_page[index + i], phys[i], -+ flags); -+ -+ phys += count; -+ vpfn += count; -+ nr -= count; -+ -+ kbase_mmu_sync_pgd(kctx->kbdev, -+ kbase_dma_addr(p) + (index * sizeof(u64)), -+ count * sizeof(u64)); -+ -+ kunmap(pfn_to_page(PFN_DOWN(pgd))); -+ } -+ -+ mutex_unlock(&kctx->mmu_lock); -+ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); -+ return 0; -+ -+fail_unlock: -+ mutex_unlock(&kctx->mmu_lock); -+ kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); -+ return err; -+} -+ -+/* This is a debug feature only */ -+static void mmu_check_unused(struct kbase_context *kctx, phys_addr_t pgd) -+{ -+ u64 *page; -+ int i; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); -+ /* kmap_atomic should NEVER fail. */ -+ KBASE_DEBUG_ASSERT(NULL != page); -+ -+ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { -+ if (kctx->kbdev->mmu_mode->ate_is_valid(page[i])) -+ beenthere(kctx, "live pte %016lx", (unsigned long)page[i]); -+ } -+ kunmap_atomic(page); -+} -+ -+static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int level, int zap, u64 *pgd_page_buffer) -+{ -+ phys_addr_t target_pgd; -+ u64 *pgd_page; -+ int i; -+ struct kbase_mmu_mode const *mmu_mode; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ lockdep_assert_held(&kctx->mmu_lock); -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); -+ /* kmap_atomic should NEVER fail. */ -+ KBASE_DEBUG_ASSERT(NULL != pgd_page); -+ /* Copy the page to our preallocated buffer so that we can minimize kmap_atomic usage */ -+ memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); -+ kunmap_atomic(pgd_page); -+ pgd_page = pgd_page_buffer; -+ -+ mmu_mode = kctx->kbdev->mmu_mode; -+ -+ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { -+ target_pgd = mmu_mode->pte_to_phy_addr(pgd_page[i]); -+ -+ if (target_pgd) { -+ if (level < (MIDGARD_MMU_BOTTOMLEVEL - 1)) { -+ mmu_teardown_level(kctx, target_pgd, level + 1, zap, pgd_page_buffer + (PAGE_SIZE / sizeof(u64))); -+ } else { -+ /* -+ * So target_pte is a level-3 page. -+ * As a leaf, it is safe to free it. -+ * Unless we have live pages attached to it! -+ */ -+ mmu_check_unused(kctx, target_pgd); -+ } -+ -+ beenthere(kctx, "pte %lx level %d", (unsigned long)target_pgd, level + 1); -+ if (zap) { -+ struct page *p = phys_to_page(target_pgd); -+ -+ kbase_mem_pool_free(&kctx->mem_pool, p, true); -+ kbase_process_page_usage_dec(kctx, 1); -+ kbase_atomic_sub_pages(1, &kctx->used_pages); -+ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); -+ } -+ } -+ } -+} -+ -+int kbase_mmu_init(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages); -+ -+ mutex_init(&kctx->mmu_lock); -+ -+ /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ -+ kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); -+ -+ if (NULL == kctx->mmu_teardown_pages) -+ return -ENOMEM; -+ -+ return 0; -+} -+ -+void kbase_mmu_term(struct kbase_context *kctx) -+{ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); -+ -+ kfree(kctx->mmu_teardown_pages); -+ kctx->mmu_teardown_pages = NULL; -+} -+ -+void kbase_mmu_free_pgd(struct kbase_context *kctx) -+{ -+ int new_page_count __maybe_unused; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); -+ -+ mutex_lock(&kctx->mmu_lock); -+ mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages); -+ mutex_unlock(&kctx->mmu_lock); -+ -+ beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd); -+ kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true); -+ kbase_process_page_usage_dec(kctx, 1); -+ new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages); -+ kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); -+ -+ KBASE_TLSTREAM_AUX_PAGESALLOC( -+ (u32)kctx->id, -+ (u64)new_page_count); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd); -+ -+static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, int level, char ** const buffer, size_t *size_left) -+{ -+ phys_addr_t target_pgd; -+ u64 *pgd_page; -+ int i; -+ size_t size = KBASE_MMU_PAGE_ENTRIES * sizeof(u64) + sizeof(u64); -+ size_t dump_size; -+ struct kbase_mmu_mode const *mmu_mode; -+ -+ KBASE_DEBUG_ASSERT(NULL != kctx); -+ lockdep_assert_held(&kctx->mmu_lock); -+ -+ mmu_mode = kctx->kbdev->mmu_mode; -+ -+ pgd_page = kmap(pfn_to_page(PFN_DOWN(pgd))); -+ if (!pgd_page) { -+ dev_warn(kctx->kbdev->dev, "kbasep_mmu_dump_level: kmap failure\n"); -+ return 0; -+ } -+ -+ if (*size_left >= size) { -+ /* A modified physical address that contains the page table level */ -+ u64 m_pgd = pgd | level; -+ -+ /* Put the modified physical address in the output buffer */ -+ memcpy(*buffer, &m_pgd, sizeof(m_pgd)); -+ *buffer += sizeof(m_pgd); -+ -+ /* Followed by the page table itself */ -+ memcpy(*buffer, pgd_page, sizeof(u64) * KBASE_MMU_PAGE_ENTRIES); -+ *buffer += sizeof(u64) * KBASE_MMU_PAGE_ENTRIES; -+ -+ *size_left -= size; -+ } -+ -+ if (level < MIDGARD_MMU_BOTTOMLEVEL) { -+ for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) { -+ if (mmu_mode->pte_is_valid(pgd_page[i])) { -+ target_pgd = mmu_mode->pte_to_phy_addr( -+ pgd_page[i]); -+ -+ dump_size = kbasep_mmu_dump_level(kctx, -+ target_pgd, level + 1, -+ buffer, size_left); -+ if (!dump_size) { -+ kunmap(pfn_to_page(PFN_DOWN(pgd))); -+ return 0; -+ } -+ size += dump_size; -+ } -+ } -+ } -+ -+ kunmap(pfn_to_page(PFN_DOWN(pgd))); -+ -+ return size; -+} -+ -+void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) -+{ -+ void *kaddr; -+ size_t size_left; -+ -+ KBASE_DEBUG_ASSERT(kctx); -+ -+ if (0 == nr_pages) { -+ /* can't dump in a 0 sized buffer, early out */ -+ return NULL; -+ } -+ -+ size_left = nr_pages * PAGE_SIZE; -+ -+ KBASE_DEBUG_ASSERT(0 != size_left); -+ kaddr = vmalloc_user(size_left); -+ -+ mutex_lock(&kctx->mmu_lock); -+ -+ if (kaddr) { -+ u64 end_marker = 0xFFULL; -+ char *buffer; -+ char *mmu_dump_buffer; -+ u64 config[3]; -+ size_t size; -+ -+ buffer = (char *)kaddr; -+ mmu_dump_buffer = buffer; -+ -+ if (kctx->api_version >= KBASE_API_VERSION(8, 4)) { -+ struct kbase_mmu_setup as_setup; -+ -+ kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup); -+ config[0] = as_setup.transtab; -+ config[1] = as_setup.memattr; -+ config[2] = as_setup.transcfg; -+ memcpy(buffer, &config, sizeof(config)); -+ mmu_dump_buffer += sizeof(config); -+ size_left -= sizeof(config); -+ } -+ -+ -+ -+ size = kbasep_mmu_dump_level(kctx, -+ kctx->pgd, -+ MIDGARD_MMU_TOPLEVEL, -+ &mmu_dump_buffer, -+ &size_left); -+ -+ if (!size) -+ goto fail_free; -+ -+ /* Add on the size for the end marker */ -+ size += sizeof(u64); -+ /* Add on the size for the config */ -+ if (kctx->api_version >= KBASE_API_VERSION(8, 4)) -+ size += sizeof(config); -+ -+ -+ if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) { -+ /* The buffer isn't big enough - free the memory and return failure */ -+ goto fail_free; -+ } -+ -+ /* Add the end marker */ -+ memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); -+ } -+ -+ mutex_unlock(&kctx->mmu_lock); -+ return kaddr; -+ -+fail_free: -+ vfree(kaddr); -+ mutex_unlock(&kctx->mmu_lock); -+ return NULL; -+} -+KBASE_EXPORT_TEST_API(kbase_mmu_dump); -+ -+void bus_fault_worker(struct work_struct *data) -+{ -+ struct kbase_as *faulting_as; -+ int as_no; -+ struct kbase_context *kctx; -+ struct kbase_device *kbdev; -+#if KBASE_GPU_RESET_EN -+ bool reset_status = false; -+#endif /* KBASE_GPU_RESET_EN */ -+ -+ faulting_as = container_of(data, struct kbase_as, work_busfault); -+ -+ as_no = faulting_as->number; -+ -+ kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); -+ -+ /* Grab the context that was already refcounted in kbase_mmu_interrupt(). -+ * Therefore, it cannot be scheduled out of this AS until we explicitly release it -+ */ -+ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as_no); -+ if (WARN_ON(!kctx)) { -+ atomic_dec(&kbdev->faults_pending); -+ return; -+ } -+ -+ if (unlikely(faulting_as->protected_mode)) -+ { -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Permission failure"); -+ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ atomic_dec(&kbdev->faults_pending); -+ return; -+ -+ } -+ -+#if KBASE_GPU_RESET_EN -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { -+ /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. -+ * We start the reset before switching to UNMAPPED to ensure that unrelated jobs -+ * are evicted from the GPU before the switch. -+ */ -+ dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); -+ reset_status = kbase_prepare_to_reset_gpu(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ -+ if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { -+ unsigned long flags; -+ -+ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ /* Set the MMU into unmapped mode */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_disable(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ /* AS transaction end */ -+ -+ kbase_mmu_hw_clear_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); -+ kbase_mmu_hw_enable_fault(kbdev, faulting_as, kctx, -+ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); -+ -+ kbase_pm_context_idle(kbdev); -+ } -+ -+#if KBASE_GPU_RESET_EN -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) -+ kbase_reset_gpu(kbdev); -+#endif /* KBASE_GPU_RESET_EN */ -+ -+ kbasep_js_runpool_release_ctx(kbdev, kctx); -+ -+ atomic_dec(&kbdev->faults_pending); -+} -+ -+const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) -+{ -+ const char *e; -+ -+ switch (exception_code) { -+ /* Non-Fault Status code */ -+ case 0x00: -+ e = "NOT_STARTED/IDLE/OK"; -+ break; -+ case 0x01: -+ e = "DONE"; -+ break; -+ case 0x02: -+ e = "INTERRUPTED"; -+ break; -+ case 0x03: -+ e = "STOPPED"; -+ break; -+ case 0x04: -+ e = "TERMINATED"; -+ break; -+ case 0x08: -+ e = "ACTIVE"; -+ break; -+ /* Job exceptions */ -+ case 0x40: -+ e = "JOB_CONFIG_FAULT"; -+ break; -+ case 0x41: -+ e = "JOB_POWER_FAULT"; -+ break; -+ case 0x42: -+ e = "JOB_READ_FAULT"; -+ break; -+ case 0x43: -+ e = "JOB_WRITE_FAULT"; -+ break; -+ case 0x44: -+ e = "JOB_AFFINITY_FAULT"; -+ break; -+ case 0x48: -+ e = "JOB_BUS_FAULT"; -+ break; -+ case 0x50: -+ e = "INSTR_INVALID_PC"; -+ break; -+ case 0x51: -+ e = "INSTR_INVALID_ENC"; -+ break; -+ case 0x52: -+ e = "INSTR_TYPE_MISMATCH"; -+ break; -+ case 0x53: -+ e = "INSTR_OPERAND_FAULT"; -+ break; -+ case 0x54: -+ e = "INSTR_TLS_FAULT"; -+ break; -+ case 0x55: -+ e = "INSTR_BARRIER_FAULT"; -+ break; -+ case 0x56: -+ e = "INSTR_ALIGN_FAULT"; -+ break; -+ case 0x58: -+ e = "DATA_INVALID_FAULT"; -+ break; -+ case 0x59: -+ e = "TILE_RANGE_FAULT"; -+ break; -+ case 0x5A: -+ e = "ADDR_RANGE_FAULT"; -+ break; -+ case 0x60: -+ e = "OUT_OF_MEMORY"; -+ break; -+ /* GPU exceptions */ -+ case 0x80: -+ e = "DELAYED_BUS_FAULT"; -+ break; -+ case 0x88: -+ e = "SHAREABILITY_FAULT"; -+ break; -+ /* MMU exceptions */ -+ case 0xC0: -+ case 0xC1: -+ case 0xC2: -+ case 0xC3: -+ case 0xC4: -+ case 0xC5: -+ case 0xC6: -+ case 0xC7: -+ e = "TRANSLATION_FAULT"; -+ break; -+ case 0xC8: -+ e = "PERMISSION_FAULT"; -+ break; -+ case 0xC9: -+ case 0xCA: -+ case 0xCB: -+ case 0xCC: -+ case 0xCD: -+ case 0xCE: -+ case 0xCF: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ e = "PERMISSION_FAULT"; -+ else -+ e = "UNKNOWN"; -+ break; -+ case 0xD0: -+ case 0xD1: -+ case 0xD2: -+ case 0xD3: -+ case 0xD4: -+ case 0xD5: -+ case 0xD6: -+ case 0xD7: -+ e = "TRANSTAB_BUS_FAULT"; -+ break; -+ case 0xD8: -+ e = "ACCESS_FLAG"; -+ break; -+ case 0xD9: -+ case 0xDA: -+ case 0xDB: -+ case 0xDC: -+ case 0xDD: -+ case 0xDE: -+ case 0xDF: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ e = "ACCESS_FLAG"; -+ else -+ e = "UNKNOWN"; -+ break; -+ case 0xE0: -+ case 0xE1: -+ case 0xE2: -+ case 0xE3: -+ case 0xE4: -+ case 0xE5: -+ case 0xE6: -+ case 0xE7: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ e = "ADDRESS_SIZE_FAULT"; -+ else -+ e = "UNKNOWN"; -+ break; -+ case 0xE8: -+ case 0xE9: -+ case 0xEA: -+ case 0xEB: -+ case 0xEC: -+ case 0xED: -+ case 0xEE: -+ case 0xEF: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ e = "MEMORY_ATTRIBUTES_FAULT"; -+ else -+ e = "UNKNOWN"; -+ break; -+ default: -+ e = "UNKNOWN"; -+ break; -+ }; -+ -+ return e; -+} -+ -+static const char *access_type_name(struct kbase_device *kbdev, -+ u32 fault_status) -+{ -+ switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { -+ case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ return "ATOMIC"; -+ else -+ return "UNKNOWN"; -+ case AS_FAULTSTATUS_ACCESS_TYPE_READ: -+ return "READ"; -+ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: -+ return "WRITE"; -+ case AS_FAULTSTATUS_ACCESS_TYPE_EX: -+ return "EXECUTE"; -+ default: -+ WARN_ON(1); -+ return NULL; -+ } -+} -+ -+/** -+ * The caller must ensure it's retained the ctx to prevent it from being scheduled out whilst it's being worked on. -+ */ -+static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, -+ struct kbase_as *as, const char *reason_str) -+{ -+ unsigned long flags; -+ int exception_type; -+ int access_type; -+ int source_id; -+ int as_no; -+ struct kbase_device *kbdev; -+ struct kbasep_js_device_data *js_devdata; -+ -+#if KBASE_GPU_RESET_EN -+ bool reset_status = false; -+#endif -+ -+ as_no = as->number; -+ kbdev = kctx->kbdev; -+ js_devdata = &kbdev->js_data; -+ -+ /* ASSERT that the context won't leave the runpool */ -+ KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); -+ -+ /* decode the fault status */ -+ exception_type = as->fault_status & 0xFF; -+ access_type = (as->fault_status >> 8) & 0x3; -+ source_id = (as->fault_status >> 16); -+ -+ /* terminal fault, print info about the fault */ -+ dev_err(kbdev->dev, -+ "Unhandled Page fault in AS%d at VA 0x%016llX\n" -+ "Reason: %s\n" -+ "raw fault status: 0x%X\n" -+ "decoded fault status: %s\n" -+ "exception type 0x%X: %s\n" -+ "access type 0x%X: %s\n" -+ "source id 0x%X\n" -+ "pid: %d\n", -+ as_no, as->fault_addr, -+ reason_str, -+ as->fault_status, -+ (as->fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), -+ exception_type, kbase_exception_name(kbdev, exception_type), -+ access_type, access_type_name(kbdev, as->fault_status), -+ source_id, -+ kctx->pid); -+ -+ /* hardware counters dump fault handling */ -+ if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && -+ (kbdev->hwcnt.backend.state == -+ KBASE_INSTR_STATE_DUMPING)) { -+ unsigned int num_core_groups = kbdev->gpu_props.num_core_groups; -+ -+ if ((as->fault_addr >= kbdev->hwcnt.addr) && -+ (as->fault_addr < (kbdev->hwcnt.addr + -+ (num_core_groups * 2048)))) -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; -+ } -+ -+ /* Stop the kctx from submitting more jobs and cause it to be scheduled -+ * out/rescheduled - this will occur on releasing the context's refcount */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbasep_js_clear_submit_allowed(js_devdata, kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* Kill any running jobs from the context. Submit is disallowed, so no more jobs from this -+ * context can appear in the job slots from this point on */ -+ kbase_backend_jm_kill_jobs_from_kctx(kctx); -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+#if KBASE_GPU_RESET_EN -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { -+ /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. -+ * We start the reset before switching to UNMAPPED to ensure that unrelated jobs -+ * are evicted from the GPU before the switch. -+ */ -+ dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery."); -+ reset_status = kbase_prepare_to_reset_gpu(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_disable(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ /* AS transaction end */ -+ /* Clear down the fault */ -+ kbase_mmu_hw_clear_fault(kbdev, as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ kbase_mmu_hw_enable_fault(kbdev, as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ -+#if KBASE_GPU_RESET_EN -+ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) -+ kbase_reset_gpu(kbdev); -+#endif /* KBASE_GPU_RESET_EN */ -+} -+ -+void kbasep_as_do_poke(struct work_struct *work) -+{ -+ struct kbase_as *as; -+ struct kbase_device *kbdev; -+ struct kbase_context *kctx; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(work); -+ as = container_of(work, struct kbase_as, poke_work); -+ kbdev = container_of(as, struct kbase_device, as[as->number]); -+ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); -+ -+ /* GPU power will already be active by virtue of the caller holding a JS -+ * reference on the address space, and will not release it until this worker -+ * has finished */ -+ -+ /* Further to the comment above, we know that while this function is running -+ * the AS will not be released as before the atom is released this workqueue -+ * is flushed (in kbase_as_poking_timer_release_atom) -+ */ -+ kctx = kbasep_js_runpool_lookup_ctx_noretain(kbdev, as->number); -+ -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ /* Force a uTLB invalidate */ -+ kbase_mmu_hw_do_operation(kbdev, as, kctx, 0, 0, -+ AS_COMMAND_UNLOCK, 0); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ /* AS transaction end */ -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (as->poke_refcount && -+ !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) { -+ /* Only queue up the timer if we need it, and we're not trying to kill it */ -+ hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer) -+{ -+ struct kbase_as *as; -+ int queue_work_ret; -+ -+ KBASE_DEBUG_ASSERT(NULL != timer); -+ as = container_of(timer, struct kbase_as, poke_timer); -+ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); -+ -+ queue_work_ret = queue_work(as->poke_wq, &as->poke_work); -+ KBASE_DEBUG_ASSERT(queue_work_ret); -+ return HRTIMER_NORESTART; -+} -+ -+/** -+ * Retain the poking timer on an atom's context (if the atom hasn't already -+ * done so), and start the timer (if it's not already started). -+ * -+ * This must only be called on a context that's scheduled in, and an atom -+ * that's running on the GPU. -+ * -+ * The caller must hold hwaccess_lock -+ * -+ * This can be called safely from atomic context -+ */ -+void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ struct kbase_as *as; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(katom); -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (katom->poking) -+ return; -+ -+ katom->poking = 1; -+ -+ /* It's safe to work on the as/as_nr without an explicit reference, -+ * because the caller holds the hwaccess_lock, and the atom itself -+ * was also running and had already taken a reference */ -+ as = &kbdev->as[kctx->as_nr]; -+ -+ if (++(as->poke_refcount) == 1) { -+ /* First refcount for poke needed: check if not already in flight */ -+ if (!as->poke_state) { -+ /* need to start poking */ -+ as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT; -+ queue_work(as->poke_wq, &as->poke_work); -+ } -+ } -+} -+ -+/** -+ * If an atom holds a poking timer, release it and wait for it to finish -+ * -+ * This must only be called on a context that's scheduled in, and an atom -+ * that still has a JS reference on the context -+ * -+ * This must \b not be called from atomic context, since it can sleep. -+ */ -+void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) -+{ -+ struct kbase_as *as; -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ KBASE_DEBUG_ASSERT(kctx); -+ KBASE_DEBUG_ASSERT(katom); -+ KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -+ -+ if (!katom->poking) -+ return; -+ -+ as = &kbdev->as[kctx->as_nr]; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ KBASE_DEBUG_ASSERT(as->poke_refcount > 0); -+ KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); -+ -+ if (--(as->poke_refcount) == 0) { -+ as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ hrtimer_cancel(&as->poke_timer); -+ flush_workqueue(as->poke_wq); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* Re-check whether it's still needed */ -+ if (as->poke_refcount) { -+ int queue_work_ret; -+ /* Poking still needed: -+ * - Another retain will not be starting the timer or queueing work, -+ * because it's still marked as in-flight -+ * - The hrtimer has finished, and has not started a new timer or -+ * queued work because it's been marked as killing -+ * -+ * So whatever happens now, just queue the work again */ -+ as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE); -+ queue_work_ret = queue_work(as->poke_wq, &as->poke_work); -+ KBASE_DEBUG_ASSERT(queue_work_ret); -+ } else { -+ /* It isn't - so mark it as not in flight, and not killing */ -+ as->poke_state = 0u; -+ -+ /* The poke associated with the atom has now finished. If this is -+ * also the last atom on the context, then we can guarentee no more -+ * pokes (and thus no more poking register accesses) will occur on -+ * the context until new atoms are run */ -+ } -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ katom->poking = 0; -+} -+ -+void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!kctx) { -+ dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Suprious IRQ or SW Design Error?\n", -+ kbase_as_has_bus_fault(as) ? "Bus error" : "Page fault", -+ as->number, as->fault_addr); -+ -+ /* Since no ctx was found, the MMU must be disabled. */ -+ WARN_ON(as->current_setup.transtab); -+ -+ if (kbase_as_has_bus_fault(as)) { -+ kbase_mmu_hw_clear_fault(kbdev, as, kctx, -+ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); -+ kbase_mmu_hw_enable_fault(kbdev, as, kctx, -+ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); -+ } else if (kbase_as_has_page_fault(as)) { -+ kbase_mmu_hw_clear_fault(kbdev, as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ kbase_mmu_hw_enable_fault(kbdev, as, kctx, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ } -+ -+#if KBASE_GPU_RESET_EN -+ if (kbase_as_has_bus_fault(as) && -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { -+ bool reset_status; -+ /* -+ * Reset the GPU, like in bus_fault_worker, in case an -+ * earlier error hasn't been properly cleared by this -+ * point. -+ */ -+ dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); -+ reset_status = kbase_prepare_to_reset_gpu_locked(kbdev); -+ if (reset_status) -+ kbase_reset_gpu_locked(kbdev); -+ } -+#endif /* KBASE_GPU_RESET_EN */ -+ -+ return; -+ } -+ -+ if (kbase_as_has_bus_fault(as)) { -+ /* -+ * hw counters dumping in progress, signal the -+ * other thread that it failed -+ */ -+ if ((kbdev->hwcnt.kctx == kctx) && -+ (kbdev->hwcnt.backend.state == -+ KBASE_INSTR_STATE_DUMPING)) -+ kbdev->hwcnt.backend.state = -+ KBASE_INSTR_STATE_FAULT; -+ -+ /* -+ * Stop the kctx from submitting more jobs and cause it -+ * to be scheduled out/rescheduled when all references -+ * to it are released -+ */ -+ kbasep_js_clear_submit_allowed(js_devdata, kctx); -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -+ dev_warn(kbdev->dev, -+ "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", -+ as->number, as->fault_addr, -+ as->fault_extra_addr); -+ else -+ dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", -+ as->number, as->fault_addr); -+ -+ /* -+ * We need to switch to UNMAPPED mode - but we do this in a -+ * worker so that we can sleep -+ */ -+ kbdev->kbase_group_error++; -+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_busfault)); -+ WARN_ON(work_pending(&as->work_busfault)); -+ queue_work(as->pf_wq, &as->work_busfault); -+ atomic_inc(&kbdev->faults_pending); -+ } else { -+ kbdev->kbase_group_error++; -+ KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&as->work_pagefault)); -+ WARN_ON(work_pending(&as->work_pagefault)); -+ queue_work(as->pf_wq, &as->work_pagefault); -+ atomic_inc(&kbdev->faults_pending); -+ } -+} -+ -+void kbase_flush_mmu_wqs(struct kbase_device *kbdev) -+{ -+ int i; -+ -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { -+ struct kbase_as *as = &kbdev->as[i]; -+ -+ flush_workqueue(as->pf_wq); -+ } -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h b/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h -new file mode 100755 -index 000000000..986e959e9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_hw.h -@@ -0,0 +1,123 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file -+ * Interface file for accessing MMU hardware functionality -+ */ -+ -+/** -+ * @page mali_kbase_mmu_hw_page MMU hardware interface -+ * -+ * @section mali_kbase_mmu_hw_intro_sec Introduction -+ * This module provides an abstraction for accessing the functionality provided -+ * by the midgard MMU and thus allows all MMU HW access to be contained within -+ * one common place and allows for different backends (implementations) to -+ * be provided. -+ */ -+ -+#ifndef _MALI_KBASE_MMU_HW_H_ -+#define _MALI_KBASE_MMU_HW_H_ -+ -+/* Forward declarations */ -+struct kbase_device; -+struct kbase_as; -+struct kbase_context; -+ -+/** -+ * @addtogroup base_kbase_api -+ * @{ -+ */ -+ -+/** -+ * @addtogroup mali_kbase_mmu_hw MMU access APIs -+ * @{ -+ */ -+ -+/** @brief MMU fault type descriptor. -+ */ -+enum kbase_mmu_fault_type { -+ KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, -+ KBASE_MMU_FAULT_TYPE_PAGE, -+ KBASE_MMU_FAULT_TYPE_BUS, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED, -+ KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED -+}; -+ -+/** @brief Configure an address space for use. -+ * -+ * Configure the MMU using the address space details setup in the -+ * @ref kbase_context structure. -+ * -+ * @param[in] kbdev kbase device to configure. -+ * @param[in] as address space to configure. -+ * @param[in] kctx kbase context to configure. -+ */ -+void kbase_mmu_hw_configure(struct kbase_device *kbdev, -+ struct kbase_as *as, struct kbase_context *kctx); -+ -+/** @brief Issue an operation to the MMU. -+ * -+ * Issue an operation (MMU invalidate, MMU flush, etc) on the address space that -+ * is associated with the provided @ref kbase_context over the specified range -+ * -+ * @param[in] kbdev kbase device to issue the MMU operation on. -+ * @param[in] as address space to issue the MMU operation on. -+ * @param[in] kctx kbase context to issue the MMU operation on. -+ * @param[in] vpfn MMU Virtual Page Frame Number to start the -+ * operation on. -+ * @param[in] nr Number of pages to work on. -+ * @param[in] type Operation type (written to ASn_COMMAND). -+ * @param[in] handling_irq Is this operation being called during the handling -+ * of an interrupt? -+ * -+ * @return Zero if the operation was successful, non-zero otherwise. -+ */ -+int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx, u64 vpfn, u32 nr, u32 type, -+ unsigned int handling_irq); -+ -+/** @brief Clear a fault that has been previously reported by the MMU. -+ * -+ * Clear a bus error or page fault that has been reported by the MMU. -+ * -+ * @param[in] kbdev kbase device to clear the fault from. -+ * @param[in] as address space to clear the fault from. -+ * @param[in] kctx kbase context to clear the fault from or NULL. -+ * @param[in] type The type of fault that needs to be cleared. -+ */ -+void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx, enum kbase_mmu_fault_type type); -+ -+/** @brief Enable fault that has been previously reported by the MMU. -+ * -+ * After a page fault or bus error has been reported by the MMU these -+ * will be disabled. After these are handled this function needs to be -+ * called to enable the page fault or bus error fault again. -+ * -+ * @param[in] kbdev kbase device to again enable the fault from. -+ * @param[in] as address space to again enable the fault from. -+ * @param[in] kctx kbase context to again enable the fault from. -+ * @param[in] type The type of fault that needs to be enabled again. -+ */ -+void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, -+ struct kbase_context *kctx, enum kbase_mmu_fault_type type); -+ -+/** @} *//* end group mali_kbase_mmu_hw */ -+/** @} *//* end group base_kbase_api */ -+ -+#endif /* _MALI_KBASE_MMU_HW_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h -new file mode 100755 -index 000000000..b487c0042 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode.h -@@ -0,0 +1,47 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _MALI_KBASE_MMU_MODE_ -+#define _MALI_KBASE_MMU_MODE_ -+ -+#include -+ -+/* Forward declarations */ -+struct kbase_context; -+struct kbase_device; -+struct kbase_as; -+struct kbase_mmu_setup; -+ -+struct kbase_mmu_mode { -+ void (*update)(struct kbase_context *kctx); -+ void (*get_as_setup)(struct kbase_context *kctx, -+ struct kbase_mmu_setup * const setup); -+ void (*disable_as)(struct kbase_device *kbdev, int as_nr); -+ phys_addr_t (*pte_to_phy_addr)(u64 entry); -+ int (*ate_is_valid)(u64 ate); -+ int (*pte_is_valid)(u64 pte); -+ void (*entry_set_ate)(u64 *entry, phys_addr_t phy, unsigned long flags); -+ void (*entry_set_pte)(u64 *entry, phys_addr_t phy); -+ void (*entry_invalidate)(u64 *entry); -+}; -+ -+struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); -+struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); -+ -+#endif /* _MALI_KBASE_MMU_MODE_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c -new file mode 100755 -index 000000000..60df17116 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_aarch64.c -@@ -0,0 +1,200 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2014, 2016, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include "mali_kbase_mmu_mode.h" -+ -+#include "mali_kbase.h" -+#include "mali_midg_regmap.h" -+ -+#define ENTRY_TYPE_MASK 3ULL -+/* For valid ATEs bit 1 = (level == 3) ? 1 : 0. -+ * The MMU is only ever configured by the driver so that ATEs -+ * are at level 3, so bit 1 should always be set -+ */ -+#define ENTRY_IS_ATE 3ULL -+#define ENTRY_IS_INVAL 2ULL -+#define ENTRY_IS_PTE 3ULL -+ -+#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ -+#define ENTRY_ACCESS_RW (1ULL << 6) /* bits 6:7 */ -+#define ENTRY_ACCESS_RO (3ULL << 6) -+#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ -+#define ENTRY_ACCESS_BIT (1ULL << 10) -+#define ENTRY_NX_BIT (1ULL << 54) -+ -+/* Helper Function to perform assignment of page table entries, to -+ * ensure the use of strd, which is required on LPAE systems. -+ */ -+static inline void page_table_entry_set(u64 *pte, u64 phy) -+{ -+#ifdef CONFIG_64BIT -+ *pte = phy; -+#elif defined(CONFIG_ARM) -+ /* -+ * In order to prevent the compiler keeping cached copies of -+ * memory, we have to explicitly say that we have updated memory. -+ * -+ * Note: We could manually move the data ourselves into R0 and -+ * R1 by specifying register variables that are explicitly -+ * given registers assignments, the down side of this is that -+ * we have to assume cpu endianness. To avoid this we can use -+ * the ldrd to read the data from memory into R0 and R1 which -+ * will respect the cpu endianness, we then use strd to make -+ * the 64 bit assignment to the page table entry. -+ */ -+ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" -+ "strd r0, r1, [%[pte]]\n\t" -+ : "=m" (*pte) -+ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) -+ : "r0", "r1"); -+#else -+#error "64-bit atomic write must be implemented for your architecture" -+#endif -+} -+ -+static void mmu_get_as_setup(struct kbase_context *kctx, -+ struct kbase_mmu_setup * const setup) -+{ -+ /* Set up the required caching policies at the correct indices -+ * in the memattr register. -+ */ -+ setup->memattr = -+ (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << -+ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | -+ (AS_MEMATTR_FORCE_TO_CACHE_ALL << -+ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | -+ (AS_MEMATTR_WRITE_ALLOC << -+ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | -+ (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << -+ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | -+ (AS_MEMATTR_AARCH64_OUTER_WA << -+ (AS_MEMATTR_INDEX_OUTER_WA * 8)); -+ -+ setup->transtab = (u64)kctx->pgd & AS_TRANSTAB_BASE_MASK; -+ setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; -+} -+ -+static void mmu_update(struct kbase_context *kctx) -+{ -+ struct kbase_device * const kbdev = kctx->kbdev; -+ struct kbase_as * const as = &kbdev->as[kctx->as_nr]; -+ struct kbase_mmu_setup * const current_setup = &as->current_setup; -+ -+ mmu_get_as_setup(kctx, current_setup); -+ -+ /* Apply the address space setting */ -+ kbase_mmu_hw_configure(kbdev, as, kctx); -+} -+ -+static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) -+{ -+ struct kbase_as * const as = &kbdev->as[as_nr]; -+ struct kbase_mmu_setup * const current_setup = &as->current_setup; -+ -+ current_setup->transtab = 0ULL; -+ current_setup->transcfg = AS_TRANSCFG_ADRMODE_UNMAPPED; -+ -+ /* Apply the address space setting */ -+ kbase_mmu_hw_configure(kbdev, as, NULL); -+} -+ -+static phys_addr_t pte_to_phy_addr(u64 entry) -+{ -+ if (!(entry & 1)) -+ return 0; -+ -+ return entry & ~0xFFF; -+} -+ -+static int ate_is_valid(u64 ate) -+{ -+ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); -+} -+ -+static int pte_is_valid(u64 pte) -+{ -+ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); -+} -+ -+/* -+ * Map KBASE_REG flags to MMU flags -+ */ -+static u64 get_mmu_flags(unsigned long flags) -+{ -+ u64 mmu_flags; -+ -+ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ -+ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; -+ -+ /* Set access flags - note that AArch64 stage 1 does not support -+ * write-only access, so we use read/write instead -+ */ -+ if (flags & KBASE_REG_GPU_WR) -+ mmu_flags |= ENTRY_ACCESS_RW; -+ else if (flags & KBASE_REG_GPU_RD) -+ mmu_flags |= ENTRY_ACCESS_RO; -+ -+ /* nx if requested */ -+ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; -+ -+ if (flags & KBASE_REG_SHARE_BOTH) { -+ /* inner and outer shareable */ -+ mmu_flags |= SHARE_BOTH_BITS; -+ } else if (flags & KBASE_REG_SHARE_IN) { -+ /* inner shareable coherency */ -+ mmu_flags |= SHARE_INNER_BITS; -+ } -+ -+ return mmu_flags; -+} -+ -+static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags) -+{ -+ page_table_entry_set(entry, (phy & ~0xFFF) | -+ get_mmu_flags(flags) | -+ ENTRY_ACCESS_BIT | ENTRY_IS_ATE); -+} -+ -+static void entry_set_pte(u64 *entry, phys_addr_t phy) -+{ -+ page_table_entry_set(entry, (phy & ~0xFFF) | -+ ENTRY_ACCESS_BIT | ENTRY_IS_PTE); -+} -+ -+static void entry_invalidate(u64 *entry) -+{ -+ page_table_entry_set(entry, ENTRY_IS_INVAL); -+} -+ -+static struct kbase_mmu_mode const aarch64_mode = { -+ .update = mmu_update, -+ .get_as_setup = mmu_get_as_setup, -+ .disable_as = mmu_disable_as, -+ .pte_to_phy_addr = pte_to_phy_addr, -+ .ate_is_valid = ate_is_valid, -+ .pte_is_valid = pte_is_valid, -+ .entry_set_ate = entry_set_ate, -+ .entry_set_pte = entry_set_pte, -+ .entry_invalidate = entry_invalidate -+}; -+ -+struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void) -+{ -+ return &aarch64_mode; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c -new file mode 100755 -index 000000000..53fbbc73a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_mmu_mode_lpae.c -@@ -0,0 +1,198 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include "mali_kbase_mmu_mode.h" -+ -+#include "mali_kbase.h" -+#include "mali_midg_regmap.h" -+ -+#define ENTRY_TYPE_MASK 3ULL -+#define ENTRY_IS_ATE 1ULL -+#define ENTRY_IS_INVAL 2ULL -+#define ENTRY_IS_PTE 3ULL -+ -+#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ -+#define ENTRY_RD_BIT (1ULL << 6) -+#define ENTRY_WR_BIT (1ULL << 7) -+#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ -+#define ENTRY_ACCESS_BIT (1ULL << 10) -+#define ENTRY_NX_BIT (1ULL << 54) -+ -+#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ -+ ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) -+ -+/* Helper Function to perform assignment of page table entries, to -+ * ensure the use of strd, which is required on LPAE systems. -+ */ -+static inline void page_table_entry_set(u64 *pte, u64 phy) -+{ -+#ifdef CONFIG_64BIT -+ *pte = phy; -+#elif defined(CONFIG_ARM) -+ /* -+ * In order to prevent the compiler keeping cached copies of -+ * memory, we have to explicitly say that we have updated -+ * memory. -+ * -+ * Note: We could manually move the data ourselves into R0 and -+ * R1 by specifying register variables that are explicitly -+ * given registers assignments, the down side of this is that -+ * we have to assume cpu endianness. To avoid this we can use -+ * the ldrd to read the data from memory into R0 and R1 which -+ * will respect the cpu endianness, we then use strd to make -+ * the 64 bit assignment to the page table entry. -+ */ -+ asm volatile("ldrd r0, r1, [%[ptemp]]\n\t" -+ "strd r0, r1, [%[pte]]\n\t" -+ : "=m" (*pte) -+ : [ptemp] "r" (&phy), [pte] "r" (pte), "m" (phy) -+ : "r0", "r1"); -+#else -+#error "64-bit atomic write must be implemented for your architecture" -+#endif -+} -+ -+static void mmu_get_as_setup(struct kbase_context *kctx, -+ struct kbase_mmu_setup * const setup) -+{ -+ /* Set up the required caching policies at the correct indices -+ * in the memattr register. */ -+ setup->memattr = -+ (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << -+ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | -+ (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << -+ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | -+ (AS_MEMATTR_LPAE_WRITE_ALLOC << -+ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | -+ (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << -+ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | -+ (AS_MEMATTR_LPAE_OUTER_WA << -+ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | -+ 0; /* The other indices are unused for now */ -+ -+ setup->transtab = ((u64)kctx->pgd & -+ ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | -+ AS_TRANSTAB_LPAE_ADRMODE_TABLE | -+ AS_TRANSTAB_LPAE_READ_INNER; -+ -+ setup->transcfg = 0; -+} -+ -+static void mmu_update(struct kbase_context *kctx) -+{ -+ struct kbase_device * const kbdev = kctx->kbdev; -+ struct kbase_as * const as = &kbdev->as[kctx->as_nr]; -+ struct kbase_mmu_setup * const current_setup = &as->current_setup; -+ -+ mmu_get_as_setup(kctx, current_setup); -+ -+ /* Apply the address space setting */ -+ kbase_mmu_hw_configure(kbdev, as, kctx); -+} -+ -+static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) -+{ -+ struct kbase_as * const as = &kbdev->as[as_nr]; -+ struct kbase_mmu_setup * const current_setup = &as->current_setup; -+ -+ current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; -+ -+ /* Apply the address space setting */ -+ kbase_mmu_hw_configure(kbdev, as, NULL); -+} -+ -+static phys_addr_t pte_to_phy_addr(u64 entry) -+{ -+ if (!(entry & 1)) -+ return 0; -+ -+ return entry & ~0xFFF; -+} -+ -+static int ate_is_valid(u64 ate) -+{ -+ return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); -+} -+ -+static int pte_is_valid(u64 pte) -+{ -+ return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); -+} -+ -+/* -+ * Map KBASE_REG flags to MMU flags -+ */ -+static u64 get_mmu_flags(unsigned long flags) -+{ -+ u64 mmu_flags; -+ -+ /* store mem_attr index as 4:2 (macro called ensures 3 bits already) */ -+ mmu_flags = KBASE_REG_MEMATTR_VALUE(flags) << 2; -+ -+ /* write perm if requested */ -+ mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; -+ /* read perm if requested */ -+ mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; -+ /* nx if requested */ -+ mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; -+ -+ if (flags & KBASE_REG_SHARE_BOTH) { -+ /* inner and outer shareable */ -+ mmu_flags |= SHARE_BOTH_BITS; -+ } else if (flags & KBASE_REG_SHARE_IN) { -+ /* inner shareable coherency */ -+ mmu_flags |= SHARE_INNER_BITS; -+ } -+ -+ return mmu_flags; -+} -+ -+static void entry_set_ate(u64 *entry, phys_addr_t phy, unsigned long flags) -+{ -+ page_table_entry_set(entry, (phy & ~0xFFF) | -+ get_mmu_flags(flags) | -+ ENTRY_IS_ATE); -+} -+ -+static void entry_set_pte(u64 *entry, phys_addr_t phy) -+{ -+ page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); -+} -+ -+static void entry_invalidate(u64 *entry) -+{ -+ page_table_entry_set(entry, ENTRY_IS_INVAL); -+} -+ -+static struct kbase_mmu_mode const lpae_mode = { -+ .update = mmu_update, -+ .get_as_setup = mmu_get_as_setup, -+ .disable_as = mmu_disable_as, -+ .pte_to_phy_addr = pte_to_phy_addr, -+ .ate_is_valid = ate_is_valid, -+ .pte_is_valid = pte_is_valid, -+ .entry_set_ate = entry_set_ate, -+ .entry_set_pte = entry_set_pte, -+ .entry_invalidate = entry_invalidate -+}; -+ -+struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) -+{ -+ return &lpae_mode; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c b/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c -new file mode 100755 -index 000000000..1a44957fe ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c -@@ -0,0 +1,124 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2014, 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+ -+#include -+#include -+#include -+#include -+#include -+ -+ -+/* -+ * This file is included only for type definitions and functions belonging to -+ * specific platform folders. Do not add dependencies with symbols that are -+ * defined somewhere else. -+ */ -+#include -+ -+#define PLATFORM_CONFIG_RESOURCE_COUNT 4 -+#define PLATFORM_CONFIG_IRQ_RES_COUNT 3 -+ -+static struct platform_device *mali_device; -+ -+#ifndef CONFIG_OF -+/** -+ * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources -+ * -+ * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function -+ * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. -+ * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. -+ * -+ * @param[in] io_resource Input IO resource data -+ * @param[out] linux_resources Pointer to output array of Linux resource structures -+ */ -+static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) -+{ -+ if (!io_resources || !linux_resources) { -+ pr_err("%s: couldn't find proper resources\n", __func__); -+ return; -+ } -+ -+ memset(linux_resources, 0, PLATFORM_CONFIG_RESOURCE_COUNT * sizeof(struct resource)); -+ -+ linux_resources[0].start = io_resources->io_memory_region.start; -+ linux_resources[0].end = io_resources->io_memory_region.end; -+ linux_resources[0].flags = IORESOURCE_MEM; -+ -+ linux_resources[1].start = io_resources->job_irq_number; -+ linux_resources[1].end = io_resources->job_irq_number; -+ linux_resources[1].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; -+ -+ linux_resources[2].start = io_resources->mmu_irq_number; -+ linux_resources[2].end = io_resources->mmu_irq_number; -+ linux_resources[2].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; -+ -+ linux_resources[3].start = io_resources->gpu_irq_number; -+ linux_resources[3].end = io_resources->gpu_irq_number; -+ linux_resources[3].flags = IORESOURCE_IRQ | IORESOURCE_IRQ_HIGHLEVEL; -+} -+#endif /* CONFIG_OF */ -+ -+int kbase_platform_fake_register(void) -+{ -+ struct kbase_platform_config *config; -+#ifndef CONFIG_OF -+ struct resource resources[PLATFORM_CONFIG_RESOURCE_COUNT]; -+#endif -+ int err; -+ -+ config = kbase_get_platform_config(); /* declared in midgard/mali_kbase_config.h but defined in platform folder */ -+ if (config == NULL) { -+ pr_err("%s: couldn't get platform config\n", __func__); -+ return -ENODEV; -+ } -+ -+ mali_device = platform_device_alloc("mali", 0); -+ if (mali_device == NULL) -+ return -ENOMEM; -+ -+#ifndef CONFIG_OF -+ kbasep_config_parse_io_resources(config->io_resources, resources); -+ err = platform_device_add_resources(mali_device, resources, PLATFORM_CONFIG_RESOURCE_COUNT); -+ if (err) { -+ platform_device_put(mali_device); -+ mali_device = NULL; -+ return err; -+ } -+#endif /* CONFIG_OF */ -+ -+ err = platform_device_add(mali_device); -+ if (err) { -+ platform_device_unregister(mali_device); -+ mali_device = NULL; -+ return err; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(kbase_platform_fake_register); -+ -+void kbase_platform_fake_unregister(void) -+{ -+ if (mali_device) -+ platform_device_unregister(mali_device); -+} -+EXPORT_SYMBOL(kbase_platform_fake_unregister); -+ -+#endif /* CONFIG_MALI_PLATFORM_FAKE */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_pm.c b/drivers/gpu/arm/midgard/mali_kbase_pm.c -new file mode 100755 -index 000000000..97d543464 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_pm.c -@@ -0,0 +1,205 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_pm.c -+ * Base kernel power management APIs -+ */ -+ -+#include -+#include -+#include -+ -+#include -+ -+int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) -+{ -+ return kbase_hwaccess_pm_powerup(kbdev, flags); -+} -+ -+void kbase_pm_halt(struct kbase_device *kbdev) -+{ -+ kbase_hwaccess_pm_halt(kbdev); -+} -+ -+void kbase_pm_context_active(struct kbase_device *kbdev) -+{ -+ (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); -+} -+ -+int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ int c; -+ int old_count; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ /* Trace timeline information about how long it took to handle the decision -+ * to powerup. Sometimes the event might be missed due to reading the count -+ * outside of mutex, but this is necessary to get the trace timing -+ * correct. */ -+ old_count = kbdev->pm.active_count; -+ if (old_count == 0) -+ kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ if (kbase_pm_is_suspending(kbdev)) { -+ switch (suspend_handler) { -+ case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: -+ if (kbdev->pm.active_count != 0) -+ break; -+ /* FALLTHROUGH */ -+ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ if (old_count == 0) -+ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); -+ return 1; -+ -+ case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE: -+ /* FALLTHROUGH */ -+ default: -+ KBASE_DEBUG_ASSERT_MSG(false, "unreachable"); -+ break; -+ } -+ } -+ c = ++kbdev->pm.active_count; -+ KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c); -+ -+ /* Trace the event being handled */ -+ if (old_count == 0) -+ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE); -+ -+ if (c == 1) -+ /* First context active: Power on the GPU and any cores requested by -+ * the policy */ -+ kbase_hwaccess_pm_gpu_active(kbdev); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ return 0; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_context_active); -+ -+void kbase_pm_context_idle(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ int c; -+ int old_count; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ -+ /* Trace timeline information about how long it took to handle the decision -+ * to powerdown. Sometimes the event might be missed due to reading the -+ * count outside of mutex, but this is necessary to get the trace timing -+ * correct. */ -+ old_count = kbdev->pm.active_count; -+ if (old_count == 0) -+ kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); -+ -+ mutex_lock(&js_devdata->runpool_mutex); -+ mutex_lock(&kbdev->pm.lock); -+ -+ c = --kbdev->pm.active_count; -+ KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, c); -+ KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c); -+ -+ KBASE_DEBUG_ASSERT(c >= 0); -+ -+ /* Trace the event being handled */ -+ if (old_count == 0) -+ kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE); -+ -+ if (c == 0) { -+ /* Last context has gone idle */ -+ kbase_hwaccess_pm_gpu_idle(kbdev); -+ -+ /* Wake up anyone waiting for this to become 0 (e.g. suspend). The -+ * waiters must synchronize with us by locking the pm.lock after -+ * waiting */ -+ wake_up(&kbdev->pm.zero_active_count_wait); -+ } -+ -+ mutex_unlock(&kbdev->pm.lock); -+ mutex_unlock(&js_devdata->runpool_mutex); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_pm_context_idle); -+ -+void kbase_pm_suspend(struct kbase_device *kbdev) -+{ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ /* Suspend vinstr. -+ * This call will block until vinstr is suspended. */ -+ kbase_vinstr_suspend(kbdev->vinstr_ctx); -+ -+ mutex_lock(&kbdev->pm.lock); -+ KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); -+ kbdev->pm.suspending = true; -+ mutex_unlock(&kbdev->pm.lock); -+ -+ /* From now on, the active count will drop towards zero. Sometimes, it'll -+ * go up briefly before going down again. However, once it reaches zero it -+ * will stay there - guaranteeing that we've idled all pm references */ -+ -+ /* Suspend job scheduler and associated components, so that it releases all -+ * the PM active count references */ -+ kbasep_js_suspend(kbdev); -+ -+ /* Wait for the active count to reach zero. This is not the same as -+ * waiting for a power down, since not all policies power down when this -+ * reaches zero. */ -+ wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); -+ -+ /* NOTE: We synchronize with anything that was just finishing a -+ * kbase_pm_context_idle() call by locking the pm.lock below */ -+ -+ kbase_hwaccess_pm_suspend(kbdev); -+} -+ -+void kbase_pm_resume(struct kbase_device *kbdev) -+{ -+ /* MUST happen before any pm_context_active calls occur */ -+ kbase_hwaccess_pm_resume(kbdev); -+ -+ /* Initial active call, to power on the GPU/cores if needed */ -+ kbase_pm_context_active(kbdev); -+ -+ /* Resume any blocked atoms (which may cause contexts to be scheduled in -+ * and dependent atoms to run) */ -+ kbase_resume_suspended_soft_jobs(kbdev); -+ -+ /* Resume the Job Scheduler and associated components, and start running -+ * atoms */ -+ kbasep_js_resume(kbdev); -+ -+ /* Matching idle call, to power off the GPU/cores if we didn't actually -+ * need it and the policy doesn't want it on */ -+ kbase_pm_context_idle(kbdev); -+ -+ /* Resume vinstr operation */ -+ kbase_vinstr_resume(kbdev->vinstr_ctx); -+} -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_pm.h b/drivers/gpu/arm/midgard/mali_kbase_pm.h -new file mode 100755 -index 000000000..37fa2479d ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_pm.h -@@ -0,0 +1,171 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_kbase_pm.h -+ * Power management API definitions -+ */ -+ -+#ifndef _KBASE_PM_H_ -+#define _KBASE_PM_H_ -+ -+#include "mali_kbase_hwaccess_pm.h" -+ -+#define PM_ENABLE_IRQS 0x01 -+#define PM_HW_ISSUES_DETECT 0x02 -+ -+ -+/** Initialize the power management framework. -+ * -+ * Must be called before any other power management function -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ * -+ * @return 0 if the power management framework was successfully initialized. -+ */ -+int kbase_pm_init(struct kbase_device *kbdev); -+ -+/** Power up GPU after all modules have been initialized and interrupt handlers installed. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ * -+ * @param flags Flags to pass on to kbase_pm_init_hw -+ * -+ * @return 0 if powerup was successful. -+ */ -+int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); -+ -+/** -+ * Halt the power management framework. -+ * Should ensure that no new interrupts are generated, -+ * but allow any currently running interrupt handlers to complete successfully. -+ * The GPU is forced off by the time this function returns, regardless of -+ * whether or not the active power policy asks for the GPU to be powered off. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_halt(struct kbase_device *kbdev); -+ -+/** Terminate the power management framework. -+ * -+ * No power management functions may be called after this -+ * (except @ref kbase_pm_init) -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_term(struct kbase_device *kbdev); -+ -+/** Increment the count of active contexts. -+ * -+ * This function should be called when a context is about to submit a job. It informs the active power policy that the -+ * GPU is going to be in use shortly and the policy is expected to start turning on the GPU. -+ * -+ * This function will block until the GPU is available. -+ * -+ * This function ASSERTS if a suspend is occuring/has occurred whilst this is -+ * in use. Use kbase_pm_contect_active_unless_suspending() instead. -+ * -+ * @note a Suspend is only visible to Kernel threads; user-space threads in a -+ * syscall cannot witness a suspend, because they are frozen before the suspend -+ * begins. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_context_active(struct kbase_device *kbdev); -+ -+ -+/** Handler codes for doing kbase_pm_context_active_handle_suspend() */ -+enum kbase_pm_suspend_handler { -+ /** A suspend is not expected/not possible - this is the same as -+ * kbase_pm_context_active() */ -+ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, -+ /** If we're suspending, fail and don't increase the active count */ -+ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE, -+ /** If we're suspending, succeed and allow the active count to increase iff -+ * it didn't go from 0->1 (i.e., we didn't re-activate the GPU). -+ * -+ * This should only be used when there is a bounded time on the activation -+ * (e.g. guarantee it's going to be idled very soon after) */ -+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE -+}; -+ -+/** Suspend 'safe' variant of kbase_pm_context_active() -+ * -+ * If a suspend is in progress, this allows for various different ways of -+ * handling the suspend. Refer to @ref enum kbase_pm_suspend_handler for details. -+ * -+ * We returns a status code indicating whether we're allowed to keep the GPU -+ * active during the suspend, depending on the handler code. If the status code -+ * indicates a failure, the caller must abort whatever operation it was -+ * attempting, and potentially queue it up for after the OS has resumed. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ * @param suspend_handler The handler code for how to handle a suspend that might occur -+ * @return zero Indicates success -+ * @return non-zero Indicates failure due to the system being suspending/suspended. -+ */ -+int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, enum kbase_pm_suspend_handler suspend_handler); -+ -+/** Decrement the reference count of active contexts. -+ * -+ * This function should be called when a context becomes idle. After this call the GPU may be turned off by the power -+ * policy so the calling code should ensure that it does not access the GPU's registers. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_context_idle(struct kbase_device *kbdev); -+ -+/** -+ * Suspend the GPU and prevent any further register accesses to it from Kernel -+ * threads. -+ * -+ * This is called in response to an OS suspend event, and calls into the various -+ * kbase components to complete the suspend. -+ * -+ * @note the mechanisms used here rely on all user-space threads being frozen -+ * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up -+ * the GPU e.g. via atom submission. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_suspend(struct kbase_device *kbdev); -+ -+/** -+ * Resume the GPU, allow register accesses to it, and resume running atoms on -+ * the GPU. -+ * -+ * This is called in response to an OS resume event, and calls into the various -+ * kbase components to complete the resume. -+ * -+ * @param kbdev The kbase device structure for the device (must be a valid pointer) -+ */ -+void kbase_pm_resume(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_vsync_callback - vsync callback -+ * -+ * @buffer_updated: 1 if a new frame was displayed, 0 otherwise -+ * @data: Pointer to the kbase device as returned by kbase_find_device() -+ * -+ * Callback function used to notify the power management code that a vsync has -+ * occurred on the display. -+ */ -+void kbase_pm_vsync_callback(int buffer_updated, void *data); -+ -+#endif /* _KBASE_PM_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h b/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h -new file mode 100755 -index 000000000..7fb674ede ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_profiling_gator_api.h -@@ -0,0 +1,40 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010, 2013 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file mali_kbase_profiling_gator_api.h -+ * Model interface -+ */ -+ -+#ifndef _KBASE_PROFILING_GATOR_API_H_ -+#define _KBASE_PROFILING_GATOR_API_H_ -+ -+/* -+ * List of possible actions to be controlled by Streamline. -+ * The following numbers are used by gator to control -+ * the frame buffer dumping and s/w counter reporting. -+ */ -+#define FBDUMP_CONTROL_ENABLE (1) -+#define FBDUMP_CONTROL_RATE (2) -+#define SW_COUNTER_ENABLE (3) -+#define FBDUMP_CONTROL_RESIZE_FACTOR (4) -+#define FBDUMP_CONTROL_MAX (5) -+#define FBDUMP_CONTROL_MIN FBDUMP_CONTROL_ENABLE -+ -+void _mali_profiling_control(u32 action, u32 value); -+ -+#endif /* _KBASE_PROFILING_GATOR_API */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c -new file mode 100755 -index 000000000..c97065006 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c -@@ -0,0 +1,130 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include "mali_kbase.h" -+ -+#include "mali_kbase_regs_history_debugfs.h" -+ -+#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) -+ -+#include -+ -+ -+static int regs_history_size_get(void *data, u64 *val) -+{ -+ struct kbase_io_history *const h = data; -+ -+ *val = h->size; -+ -+ return 0; -+} -+ -+static int regs_history_size_set(void *data, u64 val) -+{ -+ struct kbase_io_history *const h = data; -+ -+ return kbase_io_history_resize(h, (u16)val); -+} -+ -+ -+DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, -+ regs_history_size_get, -+ regs_history_size_set, -+ "%llu\n"); -+ -+ -+/** -+ * regs_history_show - show callback for the register access history file. -+ * -+ * @sfile: The debugfs entry -+ * @data: Data associated with the entry -+ * -+ * This function is called to dump all recent accesses to the GPU registers. -+ * -+ * @return 0 if successfully prints data in debugfs entry file, failure -+ * otherwise -+ */ -+static int regs_history_show(struct seq_file *sfile, void *data) -+{ -+ struct kbase_io_history *const h = sfile->private; -+ u16 i; -+ size_t iters; -+ unsigned long flags; -+ -+ if (!h->enabled) { -+ seq_puts(sfile, "The register access history is disabled\n"); -+ goto out; -+ } -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ iters = (h->size > h->count) ? h->count : h->size; -+ seq_printf(sfile, "Last %zu register accesses of %zu total:\n", iters, -+ h->count); -+ for (i = 0; i < iters; ++i) { -+ struct kbase_io_access *io = -+ &h->buf[(h->count - iters + i) % h->size]; -+ char const access = (io->addr & 1) ? 'w' : 'r'; -+ -+ seq_printf(sfile, "%6i: %c: reg 0x%p val %08x\n", i, access, -+ (void *)(io->addr & ~0x1), io->value); -+ } -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+ -+out: -+ return 0; -+} -+ -+ -+/** -+ * regs_history_open - open operation for regs_history debugfs file -+ * -+ * @in: &struct inode pointer -+ * @file: &struct file pointer -+ * -+ * @return file descriptor -+ */ -+static int regs_history_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, ®s_history_show, in->i_private); -+} -+ -+ -+static const struct file_operations regs_history_fops = { -+ .open = ®s_history_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+ -+void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, -+ kbdev->mali_debugfs_directory, -+ &kbdev->io_history.enabled); -+ debugfs_create_file("regs_history_size", S_IRUGO | S_IWUSR, -+ kbdev->mali_debugfs_directory, -+ &kbdev->io_history, ®s_history_size_fops); -+ debugfs_create_file("regs_history", S_IRUGO, -+ kbdev->mali_debugfs_directory, &kbdev->io_history, -+ ®s_history_fops); -+} -+ -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h -new file mode 100755 -index 000000000..f10837002 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h -@@ -0,0 +1,50 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * Header file for register access history support via debugfs -+ * -+ * This interface is made available via /sys/kernel/debug/mali#/regs_history*. -+ * -+ * Usage: -+ * - regs_history_enabled: whether recording of register accesses is enabled. -+ * Write 'y' to enable, 'n' to disable. -+ * - regs_history_size: size of the register history buffer, must be > 0 -+ * - regs_history: return the information about last accesses to the registers. -+ */ -+ -+#ifndef _KBASE_REGS_HISTORY_DEBUGFS_H -+#define _KBASE_REGS_HISTORY_DEBUGFS_H -+ -+struct kbase_device; -+ -+#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) -+ -+/** -+ * kbasep_regs_history_debugfs_init - add debugfs entries for register history -+ * -+ * @kbdev: Pointer to kbase_device containing the register history -+ */ -+void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); -+ -+#else /* CONFIG_DEBUG_FS */ -+ -+#define kbasep_regs_history_debugfs_init CSTD_NOP -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+#endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_replay.c b/drivers/gpu/arm/midgard/mali_kbase_replay.c -new file mode 100755 -index 000000000..84aa3316e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_replay.c -@@ -0,0 +1,1166 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file mali_kbase_replay.c -+ * Replay soft job handlers -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+#define JOB_NOT_STARTED 0 -+#define JOB_TYPE_NULL (1) -+#define JOB_TYPE_VERTEX (5) -+#define JOB_TYPE_TILER (7) -+#define JOB_TYPE_FUSED (8) -+#define JOB_TYPE_FRAGMENT (9) -+ -+#define JOB_HEADER_32_FBD_OFFSET (31*4) -+#define JOB_HEADER_64_FBD_OFFSET (44*4) -+ -+#define FBD_POINTER_MASK (~0x3f) -+ -+#define SFBD_TILER_OFFSET (48*4) -+ -+#define MFBD_TILER_OFFSET (14*4) -+ -+#define FBD_HIERARCHY_WEIGHTS 8 -+#define FBD_HIERARCHY_MASK_MASK 0x1fff -+ -+#define FBD_TYPE 1 -+ -+#define HIERARCHY_WEIGHTS 13 -+ -+#define JOB_HEADER_ID_MAX 0xffff -+ -+#define JOB_SOURCE_ID(status) (((status) >> 16) & 0xFFFF) -+#define JOB_POLYGON_LIST (0x03) -+ -+struct fragment_job { -+ struct job_descriptor_header header; -+ -+ u32 x[2]; -+ union { -+ u64 _64; -+ u32 _32; -+ } fragment_fbd; -+}; -+ -+static void dump_job_head(struct kbase_context *kctx, char *head_str, -+ struct job_descriptor_header *job) -+{ -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(kctx->kbdev->dev, "%s\n", head_str); -+ dev_dbg(kctx->kbdev->dev, -+ "addr = %p\n" -+ "exception_status = %x (Source ID: 0x%x Access: 0x%x Exception: 0x%x)\n" -+ "first_incomplete_task = %x\n" -+ "fault_pointer = %llx\n" -+ "job_descriptor_size = %x\n" -+ "job_type = %x\n" -+ "job_barrier = %x\n" -+ "_reserved_01 = %x\n" -+ "_reserved_02 = %x\n" -+ "_reserved_03 = %x\n" -+ "_reserved_04/05 = %x,%x\n" -+ "job_index = %x\n" -+ "dependencies = %x,%x\n", -+ job, job->exception_status, -+ JOB_SOURCE_ID(job->exception_status), -+ (job->exception_status >> 8) & 0x3, -+ job->exception_status & 0xFF, -+ job->first_incomplete_task, -+ job->fault_pointer, job->job_descriptor_size, -+ job->job_type, job->job_barrier, job->_reserved_01, -+ job->_reserved_02, job->_reserved_03, -+ job->_reserved_04, job->_reserved_05, -+ job->job_index, -+ job->job_dependency_index_1, -+ job->job_dependency_index_2); -+ -+ if (job->job_descriptor_size) -+ dev_dbg(kctx->kbdev->dev, "next = %llx\n", -+ job->next_job._64); -+ else -+ dev_dbg(kctx->kbdev->dev, "next = %x\n", -+ job->next_job._32); -+#endif -+} -+ -+static int kbasep_replay_reset_sfbd(struct kbase_context *kctx, -+ u64 fbd_address, u64 tiler_heap_free, -+ u16 hierarchy_mask, u32 default_weight) -+{ -+ struct { -+ u32 padding_1[1]; -+ u32 flags; -+ u64 padding_2[2]; -+ u64 heap_free_address; -+ u32 padding[8]; -+ u32 weights[FBD_HIERARCHY_WEIGHTS]; -+ } *fbd_tiler; -+ struct kbase_vmap_struct map; -+ -+ dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); -+ -+ fbd_tiler = kbase_vmap(kctx, fbd_address + SFBD_TILER_OFFSET, -+ sizeof(*fbd_tiler), &map); -+ if (!fbd_tiler) { -+ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_fbd: failed to map fbd\n"); -+ return -EINVAL; -+ } -+ -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(kctx->kbdev->dev, -+ "FBD tiler:\n" -+ "flags = %x\n" -+ "heap_free_address = %llx\n", -+ fbd_tiler->flags, fbd_tiler->heap_free_address); -+#endif -+ if (hierarchy_mask) { -+ u32 weights[HIERARCHY_WEIGHTS]; -+ u16 old_hierarchy_mask = fbd_tiler->flags & -+ FBD_HIERARCHY_MASK_MASK; -+ int i, j = 0; -+ -+ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { -+ if (old_hierarchy_mask & (1 << i)) { -+ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); -+ weights[i] = fbd_tiler->weights[j++]; -+ } else { -+ weights[i] = default_weight; -+ } -+ } -+ -+ -+ dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", -+ old_hierarchy_mask, hierarchy_mask); -+ -+ for (i = 0; i < HIERARCHY_WEIGHTS; i++) -+ dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", -+ i, weights[i]); -+ -+ j = 0; -+ -+ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { -+ if (hierarchy_mask & (1 << i)) { -+ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); -+ -+ dev_dbg(kctx->kbdev->dev, " Writing hierarchy level %02d (%08x) to %d\n", -+ i, weights[i], j); -+ -+ fbd_tiler->weights[j++] = weights[i]; -+ } -+ } -+ -+ for (; j < FBD_HIERARCHY_WEIGHTS; j++) -+ fbd_tiler->weights[j] = 0; -+ -+ fbd_tiler->flags = hierarchy_mask | (1 << 16); -+ } -+ -+ fbd_tiler->heap_free_address = tiler_heap_free; -+ -+ dev_dbg(kctx->kbdev->dev, "heap_free_address=%llx flags=%x\n", -+ fbd_tiler->heap_free_address, fbd_tiler->flags); -+ -+ kbase_vunmap(kctx, &map); -+ -+ return 0; -+} -+ -+static int kbasep_replay_reset_mfbd(struct kbase_context *kctx, -+ u64 fbd_address, u64 tiler_heap_free, -+ u16 hierarchy_mask, u32 default_weight) -+{ -+ struct kbase_vmap_struct map; -+ struct { -+ u32 padding_0; -+ u32 flags; -+ u64 padding_1[2]; -+ u64 heap_free_address; -+ u64 padding_2; -+ u32 weights[FBD_HIERARCHY_WEIGHTS]; -+ } *fbd_tiler; -+ -+ dev_dbg(kctx->kbdev->dev, "fbd_address: %llx\n", fbd_address); -+ -+ fbd_tiler = kbase_vmap(kctx, fbd_address + MFBD_TILER_OFFSET, -+ sizeof(*fbd_tiler), &map); -+ if (!fbd_tiler) { -+ dev_err(kctx->kbdev->dev, -+ "kbasep_replay_reset_fbd: failed to map fbd\n"); -+ return -EINVAL; -+ } -+ -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(kctx->kbdev->dev, "FBD tiler:\n" -+ "flags = %x\n" -+ "heap_free_address = %llx\n", -+ fbd_tiler->flags, -+ fbd_tiler->heap_free_address); -+#endif -+ if (hierarchy_mask) { -+ u32 weights[HIERARCHY_WEIGHTS]; -+ u16 old_hierarchy_mask = (fbd_tiler->flags) & -+ FBD_HIERARCHY_MASK_MASK; -+ int i, j = 0; -+ -+ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { -+ if (old_hierarchy_mask & (1 << i)) { -+ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); -+ weights[i] = fbd_tiler->weights[j++]; -+ } else { -+ weights[i] = default_weight; -+ } -+ } -+ -+ -+ dev_dbg(kctx->kbdev->dev, "Old hierarchy mask=%x New hierarchy mask=%x\n", -+ old_hierarchy_mask, hierarchy_mask); -+ -+ for (i = 0; i < HIERARCHY_WEIGHTS; i++) -+ dev_dbg(kctx->kbdev->dev, " Hierarchy weight %02d: %08x\n", -+ i, weights[i]); -+ -+ j = 0; -+ -+ for (i = 0; i < HIERARCHY_WEIGHTS; i++) { -+ if (hierarchy_mask & (1 << i)) { -+ KBASE_DEBUG_ASSERT(j < FBD_HIERARCHY_WEIGHTS); -+ -+ dev_dbg(kctx->kbdev->dev, -+ " Writing hierarchy level %02d (%08x) to %d\n", -+ i, weights[i], j); -+ -+ fbd_tiler->weights[j++] = weights[i]; -+ } -+ } -+ -+ for (; j < FBD_HIERARCHY_WEIGHTS; j++) -+ fbd_tiler->weights[j] = 0; -+ -+ fbd_tiler->flags = hierarchy_mask | (1 << 16); -+ } -+ -+ fbd_tiler->heap_free_address = tiler_heap_free; -+ -+ kbase_vunmap(kctx, &map); -+ -+ return 0; -+} -+ -+/** -+ * @brief Reset the status of an FBD pointed to by a tiler job -+ * -+ * This performs two functions : -+ * - Set the hierarchy mask -+ * - Reset the tiler free heap address -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] job_header Address of job header to reset. -+ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to -+ * @param[in] hierarchy_mask The hierarchy mask to use -+ * @param[in] default_weight Default hierarchy weight to write when no other -+ * weight is given in the FBD -+ * @param[in] job_64 true if this job is using 64-bit -+ * descriptors -+ * -+ * @return 0 on success, error code on failure -+ */ -+static int kbasep_replay_reset_tiler_job(struct kbase_context *kctx, -+ u64 job_header, u64 tiler_heap_free, -+ u16 hierarchy_mask, u32 default_weight, bool job_64) -+{ -+ struct kbase_vmap_struct map; -+ u64 fbd_address; -+ -+ if (job_64) { -+ u64 *job_ext; -+ -+ job_ext = kbase_vmap(kctx, -+ job_header + JOB_HEADER_64_FBD_OFFSET, -+ sizeof(*job_ext), &map); -+ -+ if (!job_ext) { -+ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); -+ return -EINVAL; -+ } -+ -+ fbd_address = *job_ext; -+ -+ kbase_vunmap(kctx, &map); -+ } else { -+ u32 *job_ext; -+ -+ job_ext = kbase_vmap(kctx, -+ job_header + JOB_HEADER_32_FBD_OFFSET, -+ sizeof(*job_ext), &map); -+ -+ if (!job_ext) { -+ dev_err(kctx->kbdev->dev, "kbasep_replay_reset_tiler_job: failed to map jc\n"); -+ return -EINVAL; -+ } -+ -+ fbd_address = *job_ext; -+ -+ kbase_vunmap(kctx, &map); -+ } -+ -+ if (fbd_address & FBD_TYPE) { -+ return kbasep_replay_reset_mfbd(kctx, -+ fbd_address & FBD_POINTER_MASK, -+ tiler_heap_free, -+ hierarchy_mask, -+ default_weight); -+ } else { -+ return kbasep_replay_reset_sfbd(kctx, -+ fbd_address & FBD_POINTER_MASK, -+ tiler_heap_free, -+ hierarchy_mask, -+ default_weight); -+ } -+} -+ -+/** -+ * @brief Reset the status of a job -+ * -+ * This performs the following functions : -+ * -+ * - Reset the Job Status field of each job to NOT_STARTED. -+ * - Set the Job Type field of any Vertex Jobs to Null Job. -+ * - For any jobs using an FBD, set the Tiler Heap Free field to the value of -+ * the tiler_heap_free parameter, and set the hierarchy level mask to the -+ * hier_mask parameter. -+ * - Offset HW dependencies by the hw_job_id_offset parameter -+ * - Set the Perform Job Barrier flag if this job is the first in the chain -+ * - Read the address of the next job header -+ * -+ * @param[in] kctx Context pointer -+ * @param[in,out] job_header Address of job header to reset. Set to address -+ * of next job header on exit. -+ * @param[in] prev_jc Previous job chain to link to, if this job is -+ * the last in the chain. -+ * @param[in] hw_job_id_offset Offset for HW job IDs -+ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to -+ * @param[in] hierarchy_mask The hierarchy mask to use -+ * @param[in] default_weight Default hierarchy weight to write when no other -+ * weight is given in the FBD -+ * @param[in] first_in_chain true if this job is the first in the chain -+ * @param[in] fragment_chain true if this job is in the fragment chain -+ * -+ * @return 0 on success, error code on failure -+ */ -+static int kbasep_replay_reset_job(struct kbase_context *kctx, -+ u64 *job_header, u64 prev_jc, -+ u64 tiler_heap_free, u16 hierarchy_mask, -+ u32 default_weight, u16 hw_job_id_offset, -+ bool first_in_chain, bool fragment_chain) -+{ -+ struct fragment_job *frag_job; -+ struct job_descriptor_header *job; -+ u64 new_job_header; -+ struct kbase_vmap_struct map; -+ -+ frag_job = kbase_vmap(kctx, *job_header, sizeof(*frag_job), &map); -+ if (!frag_job) { -+ dev_err(kctx->kbdev->dev, -+ "kbasep_replay_parse_jc: failed to map jc\n"); -+ return -EINVAL; -+ } -+ job = &frag_job->header; -+ -+ dump_job_head(kctx, "Job header:", job); -+ -+ if (job->exception_status == JOB_NOT_STARTED && !fragment_chain) { -+ dev_err(kctx->kbdev->dev, "Job already not started\n"); -+ goto out_unmap; -+ } -+ job->exception_status = JOB_NOT_STARTED; -+ -+ if (job->job_type == JOB_TYPE_VERTEX) -+ job->job_type = JOB_TYPE_NULL; -+ -+ if (job->job_type == JOB_TYPE_FUSED) { -+ dev_err(kctx->kbdev->dev, "Fused jobs can not be replayed\n"); -+ goto out_unmap; -+ } -+ -+ if (first_in_chain) -+ job->job_barrier = 1; -+ -+ if ((job->job_dependency_index_1 + hw_job_id_offset) > -+ JOB_HEADER_ID_MAX || -+ (job->job_dependency_index_2 + hw_job_id_offset) > -+ JOB_HEADER_ID_MAX || -+ (job->job_index + hw_job_id_offset) > JOB_HEADER_ID_MAX) { -+ dev_err(kctx->kbdev->dev, -+ "Job indicies/dependencies out of valid range\n"); -+ goto out_unmap; -+ } -+ -+ if (job->job_dependency_index_1) -+ job->job_dependency_index_1 += hw_job_id_offset; -+ if (job->job_dependency_index_2) -+ job->job_dependency_index_2 += hw_job_id_offset; -+ -+ job->job_index += hw_job_id_offset; -+ -+ if (job->job_descriptor_size) { -+ new_job_header = job->next_job._64; -+ if (!job->next_job._64) -+ job->next_job._64 = prev_jc; -+ } else { -+ new_job_header = job->next_job._32; -+ if (!job->next_job._32) -+ job->next_job._32 = prev_jc; -+ } -+ dump_job_head(kctx, "Updated to:", job); -+ -+ if (job->job_type == JOB_TYPE_TILER) { -+ bool job_64 = job->job_descriptor_size != 0; -+ -+ if (kbasep_replay_reset_tiler_job(kctx, *job_header, -+ tiler_heap_free, hierarchy_mask, -+ default_weight, job_64) != 0) -+ goto out_unmap; -+ -+ } else if (job->job_type == JOB_TYPE_FRAGMENT) { -+ u64 fbd_address; -+ -+ if (job->job_descriptor_size) -+ fbd_address = frag_job->fragment_fbd._64; -+ else -+ fbd_address = (u64)frag_job->fragment_fbd._32; -+ -+ if (fbd_address & FBD_TYPE) { -+ if (kbasep_replay_reset_mfbd(kctx, -+ fbd_address & FBD_POINTER_MASK, -+ tiler_heap_free, -+ hierarchy_mask, -+ default_weight) != 0) -+ goto out_unmap; -+ } else { -+ if (kbasep_replay_reset_sfbd(kctx, -+ fbd_address & FBD_POINTER_MASK, -+ tiler_heap_free, -+ hierarchy_mask, -+ default_weight) != 0) -+ goto out_unmap; -+ } -+ } -+ -+ kbase_vunmap(kctx, &map); -+ -+ *job_header = new_job_header; -+ -+ return 0; -+ -+out_unmap: -+ kbase_vunmap(kctx, &map); -+ return -EINVAL; -+} -+ -+/** -+ * @brief Find the highest job ID in a job chain -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] jc Job chain start address -+ * @param[out] hw_job_id Highest job ID in chain -+ * -+ * @return 0 on success, error code on failure -+ */ -+static int kbasep_replay_find_hw_job_id(struct kbase_context *kctx, -+ u64 jc, u16 *hw_job_id) -+{ -+ while (jc) { -+ struct job_descriptor_header *job; -+ struct kbase_vmap_struct map; -+ -+ dev_dbg(kctx->kbdev->dev, -+ "kbasep_replay_find_hw_job_id: parsing jc=%llx\n", jc); -+ -+ job = kbase_vmap(kctx, jc, sizeof(*job), &map); -+ if (!job) { -+ dev_err(kctx->kbdev->dev, "failed to map jc\n"); -+ -+ return -EINVAL; -+ } -+ -+ if (job->job_index > *hw_job_id) -+ *hw_job_id = job->job_index; -+ -+ if (job->job_descriptor_size) -+ jc = job->next_job._64; -+ else -+ jc = job->next_job._32; -+ -+ kbase_vunmap(kctx, &map); -+ } -+ -+ return 0; -+} -+ -+/** -+ * @brief Reset the status of a number of jobs -+ * -+ * This function walks the provided job chain, and calls -+ * kbasep_replay_reset_job for each job. It also links the job chain to the -+ * provided previous job chain. -+ * -+ * The function will fail if any of the jobs passed already have status of -+ * NOT_STARTED. -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] jc Job chain to be processed -+ * @param[in] prev_jc Job chain to be added to. May be NULL -+ * @param[in] tiler_heap_free The value to reset Tiler Heap Free to -+ * @param[in] hierarchy_mask The hierarchy mask to use -+ * @param[in] default_weight Default hierarchy weight to write when no other -+ * weight is given in the FBD -+ * @param[in] hw_job_id_offset Offset for HW job IDs -+ * @param[in] fragment_chain true if this chain is the fragment chain -+ * -+ * @return 0 on success, error code otherwise -+ */ -+static int kbasep_replay_parse_jc(struct kbase_context *kctx, -+ u64 jc, u64 prev_jc, -+ u64 tiler_heap_free, u16 hierarchy_mask, -+ u32 default_weight, u16 hw_job_id_offset, -+ bool fragment_chain) -+{ -+ bool first_in_chain = true; -+ int nr_jobs = 0; -+ -+ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: jc=%llx hw_job_id=%x\n", -+ jc, hw_job_id_offset); -+ -+ while (jc) { -+ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_jc: parsing jc=%llx\n", jc); -+ -+ if (kbasep_replay_reset_job(kctx, &jc, prev_jc, -+ tiler_heap_free, hierarchy_mask, -+ default_weight, hw_job_id_offset, -+ first_in_chain, fragment_chain) != 0) -+ return -EINVAL; -+ -+ first_in_chain = false; -+ -+ nr_jobs++; -+ if (fragment_chain && -+ nr_jobs >= BASE_JD_REPLAY_F_CHAIN_JOB_LIMIT) { -+ dev_err(kctx->kbdev->dev, -+ "Exceeded maximum number of jobs in fragment chain\n"); -+ return -EINVAL; -+ } -+ } -+ -+ return 0; -+} -+ -+/** -+ * @brief Reset the status of a replay job, and set up dependencies -+ * -+ * This performs the actions to allow the replay job to be re-run following -+ * completion of the passed dependency. -+ * -+ * @param[in] katom The atom to be reset -+ * @param[in] dep_atom The dependency to be attached to the atom -+ */ -+static void kbasep_replay_reset_softjob(struct kbase_jd_atom *katom, -+ struct kbase_jd_atom *dep_atom) -+{ -+ katom->status = KBASE_JD_ATOM_STATE_QUEUED; -+ kbase_jd_katom_dep_set(&katom->dep[0], dep_atom, BASE_JD_DEP_TYPE_DATA); -+ list_add_tail(&katom->dep_item[0], &dep_atom->dep_head[0]); -+} -+ -+/** -+ * @brief Allocate an unused katom -+ * -+ * This will search the provided context for an unused katom, and will mark it -+ * as KBASE_JD_ATOM_STATE_QUEUED. -+ * -+ * If no atoms are available then the function will fail. -+ * -+ * @param[in] kctx Context pointer -+ * @return An atom ID, or -1 on failure -+ */ -+static int kbasep_allocate_katom(struct kbase_context *kctx) -+{ -+ struct kbase_jd_context *jctx = &kctx->jctx; -+ int i; -+ -+ for (i = BASE_JD_ATOM_COUNT-1; i > 0; i--) { -+ if (jctx->atoms[i].status == KBASE_JD_ATOM_STATE_UNUSED) { -+ jctx->atoms[i].status = KBASE_JD_ATOM_STATE_QUEUED; -+ dev_dbg(kctx->kbdev->dev, -+ "kbasep_allocate_katom: Allocated atom %d\n", -+ i); -+ return i; -+ } -+ } -+ -+ return -1; -+} -+ -+/** -+ * @brief Release a katom -+ * -+ * This will mark the provided atom as available, and remove any dependencies. -+ * -+ * For use on error path. -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] atom_id ID of atom to release -+ */ -+static void kbasep_release_katom(struct kbase_context *kctx, int atom_id) -+{ -+ struct kbase_jd_context *jctx = &kctx->jctx; -+ -+ dev_dbg(kctx->kbdev->dev, "kbasep_release_katom: Released atom %d\n", -+ atom_id); -+ -+ while (!list_empty(&jctx->atoms[atom_id].dep_head[0])) -+ list_del(jctx->atoms[atom_id].dep_head[0].next); -+ -+ while (!list_empty(&jctx->atoms[atom_id].dep_head[1])) -+ list_del(jctx->atoms[atom_id].dep_head[1].next); -+ -+ jctx->atoms[atom_id].status = KBASE_JD_ATOM_STATE_UNUSED; -+} -+ -+static void kbasep_replay_create_atom(struct kbase_context *kctx, -+ struct base_jd_atom_v2 *atom, -+ int atom_nr, -+ base_jd_prio prio) -+{ -+ atom->nr_extres = 0; -+ atom->extres_list.value = NULL; -+ atom->device_nr = 0; -+ atom->prio = prio; -+ atom->atom_number = atom_nr; -+ -+ base_jd_atom_dep_set(&atom->pre_dep[0], 0 , BASE_JD_DEP_TYPE_INVALID); -+ base_jd_atom_dep_set(&atom->pre_dep[1], 0 , BASE_JD_DEP_TYPE_INVALID); -+ -+ atom->udata.blob[0] = 0; -+ atom->udata.blob[1] = 0; -+} -+ -+/** -+ * @brief Create two atoms for the purpose of replaying jobs -+ * -+ * Two atoms are allocated and created. The jc pointer is not set at this -+ * stage. The second atom has a dependency on the first. The remaining fields -+ * are set up as follows : -+ * -+ * - No external resources. Any required external resources will be held by the -+ * replay atom. -+ * - device_nr is set to 0. This is not relevant as -+ * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP should not be set. -+ * - Priority is inherited from the replay job. -+ * -+ * @param[out] t_atom Atom to use for tiler jobs -+ * @param[out] f_atom Atom to use for fragment jobs -+ * @param[in] prio Priority of new atom (inherited from replay soft -+ * job) -+ * @return 0 on success, error code on failure -+ */ -+static int kbasep_replay_create_atoms(struct kbase_context *kctx, -+ struct base_jd_atom_v2 *t_atom, -+ struct base_jd_atom_v2 *f_atom, -+ base_jd_prio prio) -+{ -+ int t_atom_nr, f_atom_nr; -+ -+ t_atom_nr = kbasep_allocate_katom(kctx); -+ if (t_atom_nr < 0) { -+ dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); -+ return -EINVAL; -+ } -+ -+ f_atom_nr = kbasep_allocate_katom(kctx); -+ if (f_atom_nr < 0) { -+ dev_err(kctx->kbdev->dev, "Failed to allocate katom\n"); -+ kbasep_release_katom(kctx, t_atom_nr); -+ return -EINVAL; -+ } -+ -+ kbasep_replay_create_atom(kctx, t_atom, t_atom_nr, prio); -+ kbasep_replay_create_atom(kctx, f_atom, f_atom_nr, prio); -+ -+ base_jd_atom_dep_set(&f_atom->pre_dep[0], t_atom_nr , BASE_JD_DEP_TYPE_DATA); -+ -+ return 0; -+} -+ -+#ifdef CONFIG_MALI_DEBUG -+static void payload_dump(struct kbase_context *kctx, base_jd_replay_payload *payload) -+{ -+ u64 next; -+ -+ dev_dbg(kctx->kbdev->dev, "Tiler jc list :\n"); -+ next = payload->tiler_jc_list; -+ -+ while (next) { -+ struct kbase_vmap_struct map; -+ base_jd_replay_jc *jc_struct; -+ -+ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &map); -+ -+ if (!jc_struct) -+ return; -+ -+ dev_dbg(kctx->kbdev->dev, "* jc_struct=%p jc=%llx next=%llx\n", -+ jc_struct, jc_struct->jc, jc_struct->next); -+ -+ next = jc_struct->next; -+ -+ kbase_vunmap(kctx, &map); -+ } -+} -+#endif -+ -+/** -+ * @brief Parse a base_jd_replay_payload provided by userspace -+ * -+ * This will read the payload from userspace, and parse the job chains. -+ * -+ * @param[in] kctx Context pointer -+ * @param[in] replay_atom Replay soft job atom -+ * @param[in] t_atom Atom to use for tiler jobs -+ * @param[in] f_atom Atom to use for fragment jobs -+ * @return 0 on success, error code on failure -+ */ -+static int kbasep_replay_parse_payload(struct kbase_context *kctx, -+ struct kbase_jd_atom *replay_atom, -+ struct base_jd_atom_v2 *t_atom, -+ struct base_jd_atom_v2 *f_atom) -+{ -+ base_jd_replay_payload *payload = NULL; -+ u64 next; -+ u64 prev_jc = 0; -+ u16 hw_job_id_offset = 0; -+ int ret = -EINVAL; -+ struct kbase_vmap_struct map; -+ -+ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: replay_atom->jc = %llx sizeof(payload) = %zu\n", -+ replay_atom->jc, sizeof(payload)); -+ -+ payload = kbase_vmap(kctx, replay_atom->jc, sizeof(*payload), &map); -+ if (!payload) { -+ dev_err(kctx->kbdev->dev, "kbasep_replay_parse_payload: failed to map payload into kernel space\n"); -+ return -EINVAL; -+ } -+ -+#ifdef BASE_LEGACY_UK10_2_SUPPORT -+ if (KBASE_API_VERSION(10, 3) > replay_atom->kctx->api_version) { -+ base_jd_replay_payload_uk10_2 *payload_uk10_2; -+ u16 tiler_core_req; -+ u16 fragment_core_req; -+ -+ payload_uk10_2 = (base_jd_replay_payload_uk10_2 *) payload; -+ memcpy(&tiler_core_req, &payload_uk10_2->tiler_core_req, -+ sizeof(tiler_core_req)); -+ memcpy(&fragment_core_req, &payload_uk10_2->fragment_core_req, -+ sizeof(fragment_core_req)); -+ payload->tiler_core_req = (u32)(tiler_core_req & 0x7fff); -+ payload->fragment_core_req = (u32)(fragment_core_req & 0x7fff); -+ } -+#endif /* BASE_LEGACY_UK10_2_SUPPORT */ -+ -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(kctx->kbdev->dev, "kbasep_replay_parse_payload: payload=%p\n", payload); -+ dev_dbg(kctx->kbdev->dev, "Payload structure:\n" -+ "tiler_jc_list = %llx\n" -+ "fragment_jc = %llx\n" -+ "tiler_heap_free = %llx\n" -+ "fragment_hierarchy_mask = %x\n" -+ "tiler_hierarchy_mask = %x\n" -+ "hierarchy_default_weight = %x\n" -+ "tiler_core_req = %x\n" -+ "fragment_core_req = %x\n", -+ payload->tiler_jc_list, -+ payload->fragment_jc, -+ payload->tiler_heap_free, -+ payload->fragment_hierarchy_mask, -+ payload->tiler_hierarchy_mask, -+ payload->hierarchy_default_weight, -+ payload->tiler_core_req, -+ payload->fragment_core_req); -+ payload_dump(kctx, payload); -+#endif -+ t_atom->core_req = payload->tiler_core_req | BASEP_JD_REQ_EVENT_NEVER; -+ f_atom->core_req = payload->fragment_core_req | BASEP_JD_REQ_EVENT_NEVER; -+ -+ /* Sanity check core requirements*/ -+ if ((t_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_T || -+ (f_atom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_FS || -+ t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES || -+ f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { -+ -+ int t_atom_type = t_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP; -+ int f_atom_type = f_atom->core_req & BASE_JD_REQ_ATOM_TYPE & ~BASE_JD_REQ_COHERENT_GROUP & ~BASE_JD_REQ_FS_AFBC; -+ int t_has_ex_res = t_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; -+ int f_has_ex_res = f_atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES; -+ -+ if (t_atom_type != BASE_JD_REQ_T) { -+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom not a tiler job. Was: 0x%x\n Expected: 0x%x", -+ t_atom_type, BASE_JD_REQ_T); -+ } -+ if (f_atom_type != BASE_JD_REQ_FS) { -+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom not a fragment shader. Was 0x%x Expected: 0x%x\n", -+ f_atom_type, BASE_JD_REQ_FS); -+ } -+ if (t_has_ex_res) { -+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Tiler atom has external resources.\n"); -+ } -+ if (f_has_ex_res) { -+ dev_err(kctx->kbdev->dev, "Invalid core requirement: Fragment shader atom has external resources.\n"); -+ } -+ -+ goto out; -+ } -+ -+ /* Process tiler job chains */ -+ next = payload->tiler_jc_list; -+ if (!next) { -+ dev_err(kctx->kbdev->dev, "Invalid tiler JC list\n"); -+ goto out; -+ } -+ -+ while (next) { -+ base_jd_replay_jc *jc_struct; -+ struct kbase_vmap_struct jc_map; -+ u64 jc; -+ -+ jc_struct = kbase_vmap(kctx, next, sizeof(*jc_struct), &jc_map); -+ -+ if (!jc_struct) { -+ dev_err(kctx->kbdev->dev, "Failed to map jc struct\n"); -+ goto out; -+ } -+ -+ jc = jc_struct->jc; -+ next = jc_struct->next; -+ if (next) -+ jc_struct->jc = 0; -+ -+ kbase_vunmap(kctx, &jc_map); -+ -+ if (jc) { -+ u16 max_hw_job_id = 0; -+ -+ if (kbasep_replay_find_hw_job_id(kctx, jc, -+ &max_hw_job_id) != 0) -+ goto out; -+ -+ if (kbasep_replay_parse_jc(kctx, jc, prev_jc, -+ payload->tiler_heap_free, -+ payload->tiler_hierarchy_mask, -+ payload->hierarchy_default_weight, -+ hw_job_id_offset, false) != 0) { -+ goto out; -+ } -+ -+ hw_job_id_offset += max_hw_job_id; -+ -+ prev_jc = jc; -+ } -+ } -+ t_atom->jc = prev_jc; -+ -+ /* Process fragment job chain */ -+ f_atom->jc = payload->fragment_jc; -+ if (kbasep_replay_parse_jc(kctx, payload->fragment_jc, 0, -+ payload->tiler_heap_free, -+ payload->fragment_hierarchy_mask, -+ payload->hierarchy_default_weight, 0, -+ true) != 0) { -+ goto out; -+ } -+ -+ if (!t_atom->jc || !f_atom->jc) { -+ dev_err(kctx->kbdev->dev, "Invalid payload\n"); -+ goto out; -+ } -+ -+ dev_dbg(kctx->kbdev->dev, "t_atom->jc=%llx f_atom->jc=%llx\n", -+ t_atom->jc, f_atom->jc); -+ ret = 0; -+ -+out: -+ kbase_vunmap(kctx, &map); -+ -+ return ret; -+} -+ -+static void kbase_replay_process_worker(struct work_struct *data) -+{ -+ struct kbase_jd_atom *katom; -+ struct kbase_context *kctx; -+ struct kbase_jd_context *jctx; -+ bool need_to_try_schedule_context = false; -+ -+ struct base_jd_atom_v2 t_atom, f_atom; -+ struct kbase_jd_atom *t_katom, *f_katom; -+ base_jd_prio atom_prio; -+ -+ katom = container_of(data, struct kbase_jd_atom, work); -+ kctx = katom->kctx; -+ jctx = &kctx->jctx; -+ -+ mutex_lock(&jctx->lock); -+ -+ atom_prio = kbasep_js_sched_prio_to_atom_prio(katom->sched_priority); -+ -+ if (kbasep_replay_create_atoms( -+ kctx, &t_atom, &f_atom, atom_prio) != 0) { -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ goto out; -+ } -+ -+ t_katom = &jctx->atoms[t_atom.atom_number]; -+ f_katom = &jctx->atoms[f_atom.atom_number]; -+ -+ if (kbasep_replay_parse_payload(kctx, katom, &t_atom, &f_atom) != 0) { -+ kbasep_release_katom(kctx, t_atom.atom_number); -+ kbasep_release_katom(kctx, f_atom.atom_number); -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ goto out; -+ } -+ -+ kbasep_replay_reset_softjob(katom, f_katom); -+ -+ need_to_try_schedule_context |= jd_submit_atom(kctx, &t_atom, t_katom); -+ if (t_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { -+ dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); -+ kbasep_release_katom(kctx, f_atom.atom_number); -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ goto out; -+ } -+ need_to_try_schedule_context |= jd_submit_atom(kctx, &f_atom, f_katom); -+ if (f_katom->event_code == BASE_JD_EVENT_JOB_INVALID) { -+ dev_err(kctx->kbdev->dev, "Replay failed to submit atom\n"); -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ goto out; -+ } -+ -+ katom->event_code = BASE_JD_EVENT_DONE; -+ -+out: -+ if (katom->event_code != BASE_JD_EVENT_DONE) { -+ kbase_disjoint_state_down(kctx->kbdev); -+ -+ need_to_try_schedule_context |= jd_done_nolock(katom, NULL); -+ } -+ -+ if (need_to_try_schedule_context) -+ kbase_js_sched_all(kctx->kbdev); -+ -+ mutex_unlock(&jctx->lock); -+} -+ -+/** -+ * @brief Check job replay fault -+ * -+ * This will read the job payload, checks fault type and source, then decides -+ * whether replay is required. -+ * -+ * @param[in] katom The atom to be processed -+ * @return true (success) if replay required or false on failure. -+ */ -+static bool kbase_replay_fault_check(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct device *dev = kctx->kbdev->dev; -+ base_jd_replay_payload *payload; -+ u64 job_header; -+ u64 job_loop_detect; -+ struct job_descriptor_header *job; -+ struct kbase_vmap_struct job_map; -+ struct kbase_vmap_struct map; -+ bool err = false; -+ -+ /* Replay job if fault is of type BASE_JD_EVENT_JOB_WRITE_FAULT or -+ * if force_replay is enabled. -+ */ -+ if (BASE_JD_EVENT_TERMINATED == katom->event_code) { -+ return false; -+ } else if (BASE_JD_EVENT_JOB_WRITE_FAULT == katom->event_code) { -+ return true; -+ } else if (BASE_JD_EVENT_FORCE_REPLAY == katom->event_code) { -+ katom->event_code = BASE_JD_EVENT_DATA_INVALID_FAULT; -+ return true; -+ } else if (BASE_JD_EVENT_DATA_INVALID_FAULT != katom->event_code) { -+ /* No replay for faults of type other than -+ * BASE_JD_EVENT_DATA_INVALID_FAULT. -+ */ -+ return false; -+ } -+ -+ /* Job fault is BASE_JD_EVENT_DATA_INVALID_FAULT, now scan fragment jc -+ * to find out whether the source of exception is POLYGON_LIST. Replay -+ * is required if the source of fault is POLYGON_LIST. -+ */ -+ payload = kbase_vmap(kctx, katom->jc, sizeof(*payload), &map); -+ if (!payload) { -+ dev_err(dev, "kbase_replay_fault_check: failed to map payload.\n"); -+ return false; -+ } -+ -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(dev, "kbase_replay_fault_check: payload=%p\n", payload); -+ dev_dbg(dev, "\nPayload structure:\n" -+ "fragment_jc = 0x%llx\n" -+ "fragment_hierarchy_mask = 0x%x\n" -+ "fragment_core_req = 0x%x\n", -+ payload->fragment_jc, -+ payload->fragment_hierarchy_mask, -+ payload->fragment_core_req); -+#endif -+ /* Process fragment job chain */ -+ job_header = (u64) payload->fragment_jc; -+ job_loop_detect = job_header; -+ while (job_header) { -+ job = kbase_vmap(kctx, job_header, sizeof(*job), &job_map); -+ if (!job) { -+ dev_err(dev, "failed to map jc\n"); -+ /* unmap payload*/ -+ kbase_vunmap(kctx, &map); -+ return false; -+ } -+ -+ -+ dump_job_head(kctx, "\njob_head structure:\n", job); -+ -+ /* Replay only when the polygon list reader caused the -+ * DATA_INVALID_FAULT */ -+ if ((BASE_JD_EVENT_DATA_INVALID_FAULT == katom->event_code) && -+ (JOB_POLYGON_LIST == JOB_SOURCE_ID(job->exception_status))) { -+ err = true; -+ kbase_vunmap(kctx, &job_map); -+ break; -+ } -+ -+ /* Move on to next fragment job in the list */ -+ if (job->job_descriptor_size) -+ job_header = job->next_job._64; -+ else -+ job_header = job->next_job._32; -+ -+ kbase_vunmap(kctx, &job_map); -+ -+ /* Job chain loop detected */ -+ if (job_header == job_loop_detect) -+ break; -+ } -+ -+ /* unmap payload*/ -+ kbase_vunmap(kctx, &map); -+ -+ return err; -+} -+ -+ -+/** -+ * @brief Process a replay job -+ * -+ * Called from kbase_process_soft_job. -+ * -+ * On exit, if the job has completed, katom->event_code will have been updated. -+ * If the job has not completed, and is replaying jobs, then the atom status -+ * will have been reset to KBASE_JD_ATOM_STATE_QUEUED. -+ * -+ * @param[in] katom The atom to be processed -+ * @return false if the atom has completed -+ * true if the atom is replaying jobs -+ */ -+bool kbase_replay_process(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ -+ /* Don't replay this atom if these issues are not present in the -+ * hardware */ -+ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11020) && -+ !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11024)) { -+ dev_dbg(kbdev->dev, "Hardware does not need replay workaround"); -+ -+ /* Signal failure to userspace */ -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ -+ return false; -+ } -+ -+ if (katom->event_code == BASE_JD_EVENT_DONE) { -+ dev_dbg(kbdev->dev, "Previous job succeeded - not replaying\n"); -+ -+ if (katom->retry_count) -+ kbase_disjoint_state_down(kbdev); -+ -+ return false; -+ } -+ -+ if (kbase_ctx_flag(kctx, KCTX_DYING)) { -+ dev_dbg(kbdev->dev, "Not replaying; context is dying\n"); -+ -+ if (katom->retry_count) -+ kbase_disjoint_state_down(kbdev); -+ -+ return false; -+ } -+ -+ /* Check job exception type and source before replaying. */ -+ if (!kbase_replay_fault_check(katom)) { -+ dev_dbg(kbdev->dev, -+ "Replay cancelled on event %x\n", katom->event_code); -+ /* katom->event_code is already set to the failure code of the -+ * previous job. -+ */ -+ return false; -+ } -+ -+ dev_warn(kbdev->dev, "Replaying jobs retry=%d\n", -+ katom->retry_count); -+ -+ katom->retry_count++; -+ -+ if (katom->retry_count > BASEP_JD_REPLAY_LIMIT) { -+ dev_err(kbdev->dev, "Replay exceeded limit - failing jobs\n"); -+ -+ kbase_disjoint_state_down(kbdev); -+ -+ /* katom->event_code is already set to the failure code of the -+ previous job */ -+ return false; -+ } -+ -+ /* only enter the disjoint state once for the whole time while the replay is ongoing */ -+ if (katom->retry_count == 1) -+ kbase_disjoint_state_up(kbdev); -+ -+ INIT_WORK(&katom->work, kbase_replay_process_worker); -+ queue_work(kctx->event_workq, &katom->work); -+ -+ return true; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_smc.c b/drivers/gpu/arm/midgard/mali_kbase_smc.c -new file mode 100755 -index 000000000..6c8cf73ae ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_smc.c -@@ -0,0 +1,86 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifdef CONFIG_ARM64 -+ -+#include -+#include -+ -+#include -+ -+/* __asmeq is not available on Kernel versions >= 4.20 */ -+#ifndef __asmeq -+/* -+ * This is used to ensure the compiler did actually allocate the register we -+ * asked it for some inline assembly sequences. Apparently we can't trust the -+ * compiler from one version to another so a bit of paranoia won't hurt. This -+ * string is meant to be concatenated with the inline asm string and will -+ * cause compilation to stop on mismatch. (for details, see gcc PR 15089) -+ */ -+#define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" -+#endif -+ -+static noinline u64 invoke_smc_fid(u64 function_id, -+ u64 arg0, u64 arg1, u64 arg2) -+{ -+ register u64 x0 asm("x0") = function_id; -+ register u64 x1 asm("x1") = arg0; -+ register u64 x2 asm("x2") = arg1; -+ register u64 x3 asm("x3") = arg2; -+ -+ asm volatile( -+ __asmeq("%0", "x0") -+ __asmeq("%1", "x1") -+ __asmeq("%2", "x2") -+ __asmeq("%3", "x3") -+ "smc #0\n" -+ : "+r" (x0) -+ : "r" (x1), "r" (x2), "r" (x3)); -+ -+ return x0; -+} -+ -+u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2) -+{ -+ /* Is fast call (bit 31 set) */ -+ KBASE_DEBUG_ASSERT(fid & ~SMC_FAST_CALL); -+ /* bits 16-23 must be zero for fast calls */ -+ KBASE_DEBUG_ASSERT((fid & (0xFF << 16)) == 0); -+ -+ return invoke_smc_fid(fid, arg0, arg1, arg2); -+} -+ -+u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, -+ u64 arg0, u64 arg1, u64 arg2) -+{ -+ u32 fid = 0; -+ -+ /* Only the six bits allowed should be used. */ -+ KBASE_DEBUG_ASSERT((oen & ~SMC_OEN_MASK) == 0); -+ -+ fid |= SMC_FAST_CALL; /* Bit 31: Fast call */ -+ if (smc64) -+ fid |= SMC_64; /* Bit 30: 1=SMC64, 0=SMC32 */ -+ fid |= oen; /* Bit 29:24: OEN */ -+ /* Bit 23:16: Must be zero for fast calls */ -+ fid |= (function_number); /* Bit 15:0: function number */ -+ -+ return kbase_invoke_smc_fid(fid, arg0, arg1, arg2); -+} -+ -+#endif /* CONFIG_ARM64 */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_smc.h b/drivers/gpu/arm/midgard/mali_kbase_smc.h -new file mode 100755 -index 000000000..9bff3d2e8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_smc.h -@@ -0,0 +1,67 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_SMC_H_ -+#define _KBASE_SMC_H_ -+ -+#ifdef CONFIG_ARM64 -+ -+#include -+ -+#define SMC_FAST_CALL (1 << 31) -+#define SMC_64 (1 << 30) -+ -+#define SMC_OEN_OFFSET 24 -+#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ -+#define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) -+#define SMC_OEN_STD (4 << SMC_OEN_OFFSET) -+ -+ -+/** -+ * kbase_invoke_smc_fid - Perform a secure monitor call -+ * @fid: The SMC function to call, see SMC Calling convention. -+ * @arg0: First argument to the SMC. -+ * @arg1: Second argument to the SMC. -+ * @arg2: Third argument to the SMC. -+ * -+ * See SMC Calling Convention for details. -+ * -+ * Return: the return value from the SMC. -+ */ -+u64 kbase_invoke_smc_fid(u32 fid, u64 arg0, u64 arg1, u64 arg2); -+ -+/** -+ * kbase_invoke_smc_fid - Perform a secure monitor call -+ * @oen: Owning Entity number (SIP, STD etc). -+ * @function_number: The function number within the OEN. -+ * @smc64: use SMC64 calling convention instead of SMC32. -+ * @arg0: First argument to the SMC. -+ * @arg1: Second argument to the SMC. -+ * @arg2: Third argument to the SMC. -+ * -+ * See SMC Calling Convention for details. -+ * -+ * Return: the return value from the SMC call. -+ */ -+u64 kbase_invoke_smc(u32 oen, u16 function_number, bool smc64, -+ u64 arg0, u64 arg1, u64 arg2); -+ -+#endif /* CONFIG_ARM64 */ -+ -+#endif /* _KBASE_SMC_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_softjobs.c b/drivers/gpu/arm/midgard/mali_kbase_softjobs.c -new file mode 100755 -index 000000000..396953e78 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_softjobs.c -@@ -0,0 +1,1549 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+ -+#if defined(CONFIG_DMA_SHARED_BUFFER) -+#include -+#include -+#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+#include -+#endif -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/* Mask to check cache alignment of data structures */ -+#define KBASE_CACHE_ALIGNMENT_MASK ((1<kctx; -+ unsigned long lflags; -+ -+ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); -+ list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); -+ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -+} -+ -+void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ unsigned long lflags; -+ -+ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); -+ list_del(&katom->queue); -+ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -+} -+ -+static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ /* Record the start time of this atom so we could cancel it at -+ * the right time. -+ */ -+ katom->start_timestamp = ktime_get(); -+ -+ /* Add the atom to the waiting list before the timer is -+ * (re)started to make sure that it gets processed. -+ */ -+ kbasep_add_waiting_soft_job(katom); -+ -+ /* Schedule timeout of this atom after a period if it is not active */ -+ if (!timer_pending(&kctx->soft_job_timeout)) { -+ int timeout_ms = atomic_read( -+ &kctx->kbdev->js_data.soft_job_timeout_ms); -+ mod_timer(&kctx->soft_job_timeout, -+ jiffies + msecs_to_jiffies(timeout_ms)); -+ } -+} -+ -+static int kbasep_read_soft_event_status( -+ struct kbase_context *kctx, u64 evt, unsigned char *status) -+{ -+ unsigned char *mapped_evt; -+ struct kbase_vmap_struct map; -+ -+ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); -+ if (!mapped_evt) -+ return -EFAULT; -+ -+ *status = *mapped_evt; -+ -+ kbase_vunmap(kctx, &map); -+ -+ return 0; -+} -+ -+static int kbasep_write_soft_event_status( -+ struct kbase_context *kctx, u64 evt, unsigned char new_status) -+{ -+ unsigned char *mapped_evt; -+ struct kbase_vmap_struct map; -+ -+ if ((new_status != BASE_JD_SOFT_EVENT_SET) && -+ (new_status != BASE_JD_SOFT_EVENT_RESET)) -+ return -EINVAL; -+ -+ mapped_evt = kbase_vmap(kctx, evt, sizeof(*mapped_evt), &map); -+ if (!mapped_evt) -+ return -EFAULT; -+ -+ *mapped_evt = new_status; -+ -+ kbase_vunmap(kctx, &map); -+ -+ return 0; -+} -+ -+static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) -+{ -+ struct kbase_vmap_struct map; -+ void *user_result; -+ struct timespec64 ts; -+ struct base_dump_cpu_gpu_counters data; -+ u64 system_time; -+ u64 cycle_counter; -+ u64 jc = katom->jc; -+ struct kbase_context *kctx = katom->kctx; -+ int pm_active_err; -+ -+ memset(&data, 0, sizeof(data)); -+ -+ /* Take the PM active reference as late as possible - otherwise, it could -+ * delay suspend until we process the atom (which may be at the end of a -+ * long chain of dependencies */ -+ pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); -+ if (pm_active_err) { -+ struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; -+ -+ /* We're suspended - queue this on the list of suspended jobs -+ * Use dep_item[1], because dep_item[0] was previously in use -+ * for 'waiting_soft_jobs'. -+ */ -+ mutex_lock(&js_devdata->runpool_mutex); -+ list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ /* Also adding this to the list of waiting soft job */ -+ kbasep_add_waiting_soft_job(katom); -+ -+ return pm_active_err; -+ } -+ -+ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, -+ &ts); -+ -+ kbase_pm_context_idle(kctx->kbdev); -+ -+ data.sec = ts.tv_sec; -+ data.usec = ts.tv_nsec / 1000; -+ data.system_time = system_time; -+ data.cycle_counter = cycle_counter; -+ -+ /* Assume this atom will be cancelled until we know otherwise */ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ /* GPU_WR access is checked on the range for returning the result to -+ * userspace for the following reasons: -+ * - security, this is currently how imported user bufs are checked. -+ * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ -+ user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); -+ if (!user_result) -+ return 0; -+ -+ memcpy(user_result, &data, sizeof(data)); -+ -+ kbase_vunmap(kctx, &map); -+ -+ /* Atom was fine - mark it as done */ -+ katom->event_code = BASE_JD_EVENT_DONE; -+ -+ return 0; -+} -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+/* Called by the explicit fence mechanism when a fence wait has completed */ -+void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ mutex_lock(&kctx->jctx.lock); -+ kbasep_remove_waiting_soft_job(katom); -+ kbase_finish_soft_job(katom); -+ if (jd_done_nolock(katom, NULL)) -+ kbase_js_sched_all(kctx->kbdev); -+ mutex_unlock(&kctx->jctx.lock); -+} -+#endif -+ -+static void kbasep_soft_event_complete_job(struct work_struct *work) -+{ -+ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, -+ work); -+ struct kbase_context *kctx = katom->kctx; -+ int resched; -+ -+ mutex_lock(&kctx->jctx.lock); -+ resched = jd_done_nolock(katom, NULL); -+ mutex_unlock(&kctx->jctx.lock); -+ -+ if (resched) -+ kbase_js_sched_all(kctx->kbdev); -+} -+ -+void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) -+{ -+ int cancel_timer = 1; -+ struct list_head *entry, *tmp; -+ unsigned long lflags; -+ -+ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); -+ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { -+ struct kbase_jd_atom *katom = list_entry( -+ entry, struct kbase_jd_atom, queue); -+ -+ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+ case BASE_JD_REQ_SOFT_EVENT_WAIT: -+ if (katom->jc == evt) { -+ list_del(&katom->queue); -+ -+ katom->event_code = BASE_JD_EVENT_DONE; -+ INIT_WORK(&katom->work, -+ kbasep_soft_event_complete_job); -+ queue_work(kctx->jctx.job_done_wq, -+ &katom->work); -+ } else { -+ /* There are still other waiting jobs, we cannot -+ * cancel the timer yet. -+ */ -+ cancel_timer = 0; -+ } -+ break; -+#ifdef CONFIG_MALI_FENCE_DEBUG -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ /* Keep the timer running if fence debug is enabled and -+ * there are waiting fence jobs. -+ */ -+ cancel_timer = 0; -+ break; -+#endif -+ } -+ } -+ -+ if (cancel_timer) -+ del_timer(&kctx->soft_job_timeout); -+ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -+} -+ -+#ifdef CONFIG_MALI_FENCE_DEBUG -+static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct device *dev = kctx->kbdev->dev; -+ int i; -+ -+ for (i = 0; i < 2; i++) { -+ struct kbase_jd_atom *dep; -+ -+ list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { -+ if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || -+ dep->status == KBASE_JD_ATOM_STATE_COMPLETED) -+ continue; -+ -+ if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) -+ == BASE_JD_REQ_SOFT_FENCE_TRIGGER) { -+ /* Found blocked trigger fence. */ -+ struct kbase_sync_fence_info info; -+ -+ if (!kbase_sync_fence_in_info_get(dep, &info)) { -+ dev_warn(dev, -+ "\tVictim trigger atom %d fence [%p] %s: %s\n", -+ kbase_jd_atom_id(kctx, dep), -+ info.fence, -+ info.name, -+ kbase_sync_status_string(info.status)); -+ } -+ } -+ -+ kbase_fence_debug_check_atom(dep); -+ } -+ } -+} -+ -+static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct device *dev = katom->kctx->kbdev->dev; -+ int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); -+ unsigned long lflags; -+ struct kbase_sync_fence_info info; -+ -+ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); -+ -+ if (kbase_sync_fence_in_info_get(katom, &info)) { -+ /* Fence must have signaled just after timeout. */ -+ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -+ return; -+ } -+ -+ dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", -+ kctx->tgid, kctx->id, -+ kbase_jd_atom_id(kctx, katom), -+ info.fence, timeout_ms); -+ dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", -+ info.fence, info.name, -+ kbase_sync_status_string(info.status)); -+ -+ /* Search for blocked trigger atoms */ -+ kbase_fence_debug_check_atom(katom); -+ -+ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -+ -+ kbase_sync_fence_in_dump(katom); -+} -+ -+struct kbase_fence_debug_work { -+ struct kbase_jd_atom *katom; -+ struct work_struct work; -+}; -+ -+static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) -+{ -+ struct kbase_fence_debug_work *w = container_of(work, -+ struct kbase_fence_debug_work, work); -+ struct kbase_jd_atom *katom = w->katom; -+ struct kbase_context *kctx = katom->kctx; -+ -+ mutex_lock(&kctx->jctx.lock); -+ kbase_fence_debug_wait_timeout(katom); -+ mutex_unlock(&kctx->jctx.lock); -+ -+ kfree(w); -+} -+ -+static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) -+{ -+ struct kbase_fence_debug_work *work; -+ struct kbase_context *kctx = katom->kctx; -+ -+ /* Enqueue fence debug worker. Use job_done_wq to get -+ * debug print ordered with job completion. -+ */ -+ work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); -+ /* Ignore allocation failure. */ -+ if (work) { -+ work->katom = katom; -+ INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); -+ queue_work(kctx->jctx.job_done_wq, &work->work); -+ } -+} -+#endif /* CONFIG_MALI_FENCE_DEBUG */ -+ -+void kbasep_soft_job_timeout_worker(struct timer_list *t) -+{ -+ struct kbase_context *kctx = from_timer(kctx, t, soft_job_timeout); -+ u32 timeout_ms = (u32)atomic_read( -+ &kctx->kbdev->js_data.soft_job_timeout_ms); -+ struct timer_list *timer = &kctx->soft_job_timeout; -+ ktime_t cur_time = ktime_get(); -+ bool restarting = false; -+ unsigned long lflags; -+ struct list_head *entry, *tmp; -+ -+ spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); -+ list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { -+ struct kbase_jd_atom *katom = list_entry(entry, -+ struct kbase_jd_atom, queue); -+ s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, -+ katom->start_timestamp)); -+ -+ if (elapsed_time < (s64)timeout_ms) { -+ restarting = true; -+ continue; -+ } -+ -+ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+ case BASE_JD_REQ_SOFT_EVENT_WAIT: -+ /* Take it out of the list to ensure that it -+ * will be cancelled in all cases -+ */ -+ list_del(&katom->queue); -+ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ INIT_WORK(&katom->work, kbasep_soft_event_complete_job); -+ queue_work(kctx->jctx.job_done_wq, &katom->work); -+ break; -+#ifdef CONFIG_MALI_FENCE_DEBUG -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ kbase_fence_debug_timeout(katom); -+ break; -+#endif -+ } -+ } -+ -+ if (restarting) -+ mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); -+ spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -+} -+ -+static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ unsigned char status; -+ -+ /* The status of this soft-job is stored in jc */ -+ if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ return 0; -+ } -+ -+ if (status == BASE_JD_SOFT_EVENT_SET) -+ return 0; /* Event already set, nothing to do */ -+ -+ kbasep_add_waiting_with_timeout(katom); -+ -+ return 1; -+} -+ -+static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, -+ unsigned char new_status) -+{ -+ /* Complete jobs waiting on the same event */ -+ struct kbase_context *kctx = katom->kctx; -+ -+ if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ return; -+ } -+ -+ if (new_status == BASE_JD_SOFT_EVENT_SET) -+ kbasep_complete_triggered_soft_events(kctx, katom->jc); -+} -+ -+/** -+ * kbase_soft_event_update() - Update soft event state -+ * @kctx: Pointer to context -+ * @event: Event to update -+ * @new_status: New status value of event -+ * -+ * Update the event, and wake up any atoms waiting for the event. -+ * -+ * Return: 0 on success, a negative error code on failure. -+ */ -+int kbase_soft_event_update(struct kbase_context *kctx, -+ u64 event, -+ unsigned char new_status) -+{ -+ int err = 0; -+ -+ mutex_lock(&kctx->jctx.lock); -+ -+ if (kbasep_write_soft_event_status(kctx, event, new_status)) { -+ err = -ENOENT; -+ goto out; -+ } -+ -+ if (new_status == BASE_JD_SOFT_EVENT_SET) -+ kbasep_complete_triggered_soft_events(kctx, event); -+ -+out: -+ mutex_unlock(&kctx->jctx.lock); -+ -+ return err; -+} -+ -+static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) -+{ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ if (jd_done_nolock(katom, NULL)) -+ kbase_js_sched_all(katom->kctx->kbdev); -+} -+ -+struct kbase_debug_copy_buffer { -+ size_t size; -+ struct page **pages; -+ int nr_pages; -+ size_t offset; -+ struct kbase_mem_phy_alloc *gpu_alloc; -+ -+ struct page **extres_pages; -+ int nr_extres_pages; -+}; -+ -+static inline void free_user_buffer(struct kbase_debug_copy_buffer *buffer) -+{ -+ struct page **pages = buffer->extres_pages; -+ int nr_pages = buffer->nr_extres_pages; -+ -+ if (pages) { -+ int i; -+ -+ for (i = 0; i < nr_pages; i++) { -+ struct page *pg = pages[i]; -+ -+ if (pg) -+ put_page(pg); -+ } -+ kfree(pages); -+ } -+} -+ -+static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) -+{ -+ struct kbase_debug_copy_buffer *buffers = -+ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; -+ unsigned int i; -+ unsigned int nr = katom->nr_extres; -+ -+ if (!buffers) -+ return; -+ -+ kbase_gpu_vm_lock(katom->kctx); -+ for (i = 0; i < nr; i++) { -+ int p; -+ struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; -+ -+ if (!buffers[i].pages) -+ break; -+ for (p = 0; p < buffers[i].nr_pages; p++) { -+ struct page *pg = buffers[i].pages[p]; -+ -+ if (pg) -+ put_page(pg); -+ } -+ kfree(buffers[i].pages); -+ if (gpu_alloc) { -+ switch (gpu_alloc->type) { -+ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: -+ { -+ free_user_buffer(&buffers[i]); -+ break; -+ } -+ default: -+ /* Nothing to be done. */ -+ break; -+ } -+ kbase_mem_phy_alloc_put(gpu_alloc); -+ } -+ } -+ kbase_gpu_vm_unlock(katom->kctx); -+ kfree(buffers); -+ -+ katom->jc = 0; -+} -+ -+static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) -+{ -+ struct kbase_debug_copy_buffer *buffers; -+ struct base_jd_debug_copy_buffer *user_buffers = NULL; -+ unsigned int i; -+ unsigned int nr = katom->nr_extres; -+ int ret = 0; -+ void __user *user_structs = (void __user *)(uintptr_t)katom->jc; -+ -+ if (!user_structs) -+ return -EINVAL; -+ -+ buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); -+ if (!buffers) { -+ ret = -ENOMEM; -+ katom->jc = 0; -+ goto out_cleanup; -+ } -+ katom->jc = (u64)(uintptr_t)buffers; -+ -+ user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); -+ -+ if (!user_buffers) { -+ ret = -ENOMEM; -+ goto out_cleanup; -+ } -+ -+ ret = copy_from_user(user_buffers, user_structs, -+ sizeof(*user_buffers)*nr); -+ if (ret) -+ goto out_cleanup; -+ -+ for (i = 0; i < nr; i++) { -+ u64 addr = user_buffers[i].address; -+ u64 page_addr = addr & PAGE_MASK; -+ u64 end_page_addr = addr + user_buffers[i].size - 1; -+ u64 last_page_addr = end_page_addr & PAGE_MASK; -+ int nr_pages = (last_page_addr-page_addr)/PAGE_SIZE+1; -+ int pinned_pages; -+ struct kbase_va_region *reg; -+ struct base_external_resource user_extres; -+ -+ if (!addr) -+ continue; -+ -+ buffers[i].nr_pages = nr_pages; -+ buffers[i].offset = addr & ~PAGE_MASK; -+ if (buffers[i].offset >= PAGE_SIZE) { -+ ret = -EINVAL; -+ goto out_cleanup; -+ } -+ buffers[i].size = user_buffers[i].size; -+ -+ buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), -+ GFP_KERNEL); -+ if (!buffers[i].pages) { -+ ret = -ENOMEM; -+ goto out_cleanup; -+ } -+ -+ pinned_pages = get_user_pages_fast(page_addr, -+ nr_pages, -+ 1, /* Write */ -+ buffers[i].pages); -+ if (pinned_pages < 0) { -+ ret = pinned_pages; -+ goto out_cleanup; -+ } -+ if (pinned_pages != nr_pages) { -+ ret = -EINVAL; -+ goto out_cleanup; -+ } -+ -+ user_extres = user_buffers[i].extres; -+ if (user_extres.ext_resource == 0ULL) { -+ ret = -EINVAL; -+ goto out_cleanup; -+ } -+ -+ kbase_gpu_vm_lock(katom->kctx); -+ reg = kbase_region_tracker_find_region_enclosing_address( -+ katom->kctx, user_extres.ext_resource & -+ ~BASE_EXT_RES_ACCESS_EXCLUSIVE); -+ -+ if (NULL == reg || NULL == reg->gpu_alloc || -+ (reg->flags & KBASE_REG_FREE)) { -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ -+ buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); -+ buffers[i].nr_extres_pages = reg->nr_pages; -+ -+ if (reg->nr_pages*PAGE_SIZE != buffers[i].size) -+ dev_warn(katom->kctx->kbdev->dev, "Copy buffer is not of same size as the external resource to copy.\n"); -+ -+ switch (reg->gpu_alloc->type) { -+ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: -+ { -+ struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; -+ unsigned long nr_pages = -+ alloc->imported.user_buf.nr_pages; -+ -+ if (alloc->imported.user_buf.mm != current->mm) { -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ buffers[i].extres_pages = kcalloc(nr_pages, -+ sizeof(struct page *), GFP_KERNEL); -+ if (!buffers[i].extres_pages) { -+ ret = -ENOMEM; -+ goto out_unlock; -+ } -+ -+ ret = get_user_pages_fast( -+ alloc->imported.user_buf.address, -+ nr_pages, 0, -+ buffers[i].extres_pages); -+ if (ret != nr_pages) -+ goto out_unlock; -+ ret = 0; -+ break; -+ } -+ case KBASE_MEM_TYPE_IMPORTED_UMP: -+ { -+ dev_warn(katom->kctx->kbdev->dev, -+ "UMP is not supported for debug_copy jobs\n"); -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ default: -+ /* Nothing to be done. */ -+ break; -+ } -+ kbase_gpu_vm_unlock(katom->kctx); -+ } -+ kfree(user_buffers); -+ -+ return ret; -+ -+out_unlock: -+ kbase_gpu_vm_unlock(katom->kctx); -+ -+out_cleanup: -+ kfree(buffers); -+ kfree(user_buffers); -+ -+ /* Frees allocated memory for kbase_debug_copy_job struct, including -+ * members, and sets jc to 0 */ -+ kbase_debug_copy_finish(katom); -+ return ret; -+} -+ -+static void kbase_mem_copy_from_extres_page(struct kbase_context *kctx, -+ void *extres_page, struct page **pages, unsigned int nr_pages, -+ unsigned int *target_page_nr, size_t offset, size_t *to_copy) -+{ -+ void *target_page = kmap(pages[*target_page_nr]); -+ size_t chunk = PAGE_SIZE-offset; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ if (!target_page) { -+ *target_page_nr += 1; -+ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); -+ return; -+ } -+ -+ chunk = min(chunk, *to_copy); -+ -+ memcpy(target_page + offset, extres_page, chunk); -+ *to_copy -= chunk; -+ -+ kunmap(pages[*target_page_nr]); -+ -+ *target_page_nr += 1; -+ if (*target_page_nr >= nr_pages) -+ return; -+ -+ target_page = kmap(pages[*target_page_nr]); -+ if (!target_page) { -+ *target_page_nr += 1; -+ dev_warn(kctx->kbdev->dev, "kmap failed in debug_copy job."); -+ return; -+ } -+ -+ KBASE_DEBUG_ASSERT(target_page); -+ -+ chunk = min(offset, *to_copy); -+ memcpy(target_page, extres_page + PAGE_SIZE-offset, chunk); -+ *to_copy -= chunk; -+ -+ kunmap(pages[*target_page_nr]); -+} -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, -+ unsigned long page_num, struct page **page) -+{ -+ struct sg_table *sgt = gpu_alloc->imported.umm.sgt; -+ struct sg_page_iter sg_iter; -+ unsigned long page_index = 0; -+ -+ if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) -+ return NULL; -+ -+ if (!sgt) -+ return NULL; -+ -+ if (WARN_ON(page_num >= gpu_alloc->nents)) -+ return NULL; -+ -+ for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { -+ if (page_index == page_num) { -+ *page = sg_page_iter_page(&sg_iter); -+ -+ return kmap(*page); -+ } -+ page_index++; -+ } -+ -+ return NULL; -+} -+#endif -+ -+static int kbase_mem_copy_from_extres(struct kbase_context *kctx, -+ struct kbase_debug_copy_buffer *buf_data) -+{ -+ unsigned int i; -+ unsigned int target_page_nr = 0; -+ struct page **pages = buf_data->pages; -+ u64 offset = buf_data->offset; -+ size_t extres_size = buf_data->nr_extres_pages*PAGE_SIZE; -+ size_t to_copy = min(extres_size, buf_data->size); -+ struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; -+ int ret = 0; -+ -+ KBASE_DEBUG_ASSERT(pages != NULL); -+ -+ kbase_gpu_vm_lock(kctx); -+ if (!gpu_alloc) { -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ -+ switch (gpu_alloc->type) { -+ case KBASE_MEM_TYPE_IMPORTED_USER_BUF: -+ { -+ for (i = 0; i < buf_data->nr_extres_pages; i++) { -+ struct page *pg = buf_data->extres_pages[i]; -+ void *extres_page = kmap(pg); -+ -+ if (extres_page) -+ kbase_mem_copy_from_extres_page(kctx, -+ extres_page, pages, -+ buf_data->nr_pages, -+ &target_page_nr, -+ offset, &to_copy); -+ -+ kunmap(pg); -+ if (target_page_nr >= buf_data->nr_pages) -+ break; -+ } -+ break; -+ } -+ break; -+#ifdef CONFIG_DMA_SHARED_BUFFER -+ case KBASE_MEM_TYPE_IMPORTED_UMM: { -+ struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; -+ -+ KBASE_DEBUG_ASSERT(dma_buf != NULL); -+ KBASE_DEBUG_ASSERT(dma_buf->size == -+ buf_data->nr_extres_pages * PAGE_SIZE); -+ -+ ret = dma_buf_begin_cpu_access(dma_buf, -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) -+ 0, buf_data->nr_extres_pages*PAGE_SIZE, -+#endif -+ DMA_FROM_DEVICE); -+ if (ret) -+ goto out_unlock; -+ -+ for (i = 0; i < buf_data->nr_extres_pages; i++) { -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+ struct page *pg; -+ void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); -+#else -+ void *extres_page = dma_buf_kmap(dma_buf, i); -+#endif -+ -+ if (extres_page) -+ kbase_mem_copy_from_extres_page(kctx, -+ extres_page, pages, -+ buf_data->nr_pages, -+ &target_page_nr, -+ offset, &to_copy); -+ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+ kunmap(pg); -+#else -+ dma_buf_kunmap(dma_buf, i, extres_page); -+#endif -+ if (target_page_nr >= buf_data->nr_pages) -+ break; -+ } -+ dma_buf_end_cpu_access(dma_buf, -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) -+ 0, buf_data->nr_extres_pages*PAGE_SIZE, -+#endif -+ DMA_FROM_DEVICE); -+ break; -+ } -+#endif -+ default: -+ ret = -EINVAL; -+ } -+out_unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return ret; -+ -+} -+ -+static int kbase_debug_copy(struct kbase_jd_atom *katom) -+{ -+ struct kbase_debug_copy_buffer *buffers = -+ (struct kbase_debug_copy_buffer *)(uintptr_t)katom->jc; -+ unsigned int i; -+ -+ for (i = 0; i < katom->nr_extres; i++) { -+ int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); -+ -+ if (res) -+ return res; -+ } -+ -+ return 0; -+} -+ -+static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) -+{ -+ __user void *data = (__user void *)(uintptr_t) katom->jc; -+ struct base_jit_alloc_info *info; -+ struct kbase_context *kctx = katom->kctx; -+ int ret; -+ -+ /* Fail the job if there is no info structure */ -+ if (!data) { -+ ret = -EINVAL; -+ goto fail; -+ } -+ -+ /* Copy the information for safe access and future storage */ -+ info = kzalloc(sizeof(*info), GFP_KERNEL); -+ if (!info) { -+ ret = -ENOMEM; -+ goto fail; -+ } -+ -+ if (copy_from_user(info, data, sizeof(*info)) != 0) { -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+ /* If the ID is zero then fail the job */ -+ if (info->id == 0) { -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+ /* Sanity check that the PA fits within the VA */ -+ if (info->va_pages < info->commit_pages) { -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+ /* Ensure the GPU address is correctly aligned */ -+ if ((info->gpu_alloc_addr & 0x7) != 0) { -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+ /* Replace the user pointer with our kernel allocated info structure */ -+ katom->jc = (u64)(uintptr_t) info; -+ katom->jit_blocked = false; -+ -+ lockdep_assert_held(&kctx->jctx.lock); -+ list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); -+ -+ /* -+ * Note: -+ * The provided info->gpu_alloc_addr isn't validated here as -+ * userland can cache allocations which means that even -+ * though the region is valid it doesn't represent the -+ * same thing it used to. -+ * -+ * Complete validation of va_pages, commit_pages and extent -+ * isn't done here as it will be done during the call to -+ * kbase_mem_alloc. -+ */ -+ return 0; -+ -+free_info: -+ kfree(info); -+fail: -+ katom->jc = 0; -+ return ret; -+} -+ -+static u8 kbase_jit_free_get_id(struct kbase_jd_atom *katom) -+{ -+ if (WARN_ON(katom->core_req != BASE_JD_REQ_SOFT_JIT_FREE)) -+ return 0; -+ -+ return (u8) katom->jc; -+} -+ -+static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ struct base_jit_alloc_info *info; -+ struct kbase_va_region *reg; -+ struct kbase_vmap_struct mapping; -+ u64 *ptr, new_addr; -+ -+ if (katom->jit_blocked) { -+ list_del(&katom->queue); -+ katom->jit_blocked = false; -+ } -+ -+ info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; -+ -+ /* The JIT ID is still in use so fail the allocation */ -+ if (kctx->jit_alloc[info->id]) { -+ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; -+ return 0; -+ } -+ -+ /* Create a JIT allocation */ -+ reg = kbase_jit_allocate(kctx, info); -+ if (!reg) { -+ struct kbase_jd_atom *jit_atom; -+ bool can_block = false; -+ -+ lockdep_assert_held(&kctx->jctx.lock); -+ -+ jit_atom = list_first_entry(&kctx->jit_atoms_head, -+ struct kbase_jd_atom, jit_node); -+ -+ list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { -+ if (jit_atom == katom) -+ break; -+ if (jit_atom->core_req == BASE_JD_REQ_SOFT_JIT_FREE) { -+ u8 free_id = kbase_jit_free_get_id(jit_atom); -+ -+ if (free_id && kctx->jit_alloc[free_id]) { -+ /* A JIT free which is active and -+ * submitted before this atom -+ */ -+ can_block = true; -+ break; -+ } -+ } -+ } -+ -+ if (!can_block) { -+ /* Mark the allocation so we know it's in use even if -+ * the allocation itself fails. -+ */ -+ kctx->jit_alloc[info->id] = -+ (struct kbase_va_region *) -1; -+ -+ katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; -+ return 0; -+ } -+ -+ /* There are pending frees for an active allocation -+ * so we should wait to see whether they free the memory. -+ * Add to the beginning of the list to ensure that the atom is -+ * processed only once in kbase_jit_free_finish -+ */ -+ list_add(&katom->queue, &kctx->jit_pending_alloc); -+ katom->jit_blocked = true; -+ -+ return 1; -+ } -+ -+ /* -+ * Write the address of the JIT allocation to the user provided -+ * GPU allocation. -+ */ -+ ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), -+ &mapping); -+ if (!ptr) { -+ /* -+ * Leave the allocation "live" as the JIT free jit will be -+ * submitted anyway. -+ */ -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ return 0; -+ } -+ -+ new_addr = reg->start_pfn << PAGE_SHIFT; -+ *ptr = new_addr; -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( -+ katom, info->gpu_alloc_addr, new_addr); -+ kbase_vunmap(kctx, &mapping); -+ -+ katom->event_code = BASE_JD_EVENT_DONE; -+ -+ /* -+ * Bind it to the user provided ID. Do this last so we can check for -+ * the JIT free racing this JIT alloc job. -+ */ -+ kctx->jit_alloc[info->id] = reg; -+ -+ return 0; -+} -+ -+static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) -+{ -+ struct base_jit_alloc_info *info; -+ -+ lockdep_assert_held(&katom->kctx->jctx.lock); -+ -+ /* Remove atom from jit_atoms_head list */ -+ list_del(&katom->jit_node); -+ -+ if (katom->jit_blocked) { -+ list_del(&katom->queue); -+ katom->jit_blocked = false; -+ } -+ -+ info = (struct base_jit_alloc_info *) (uintptr_t) katom->jc; -+ /* Free the info structure */ -+ kfree(info); -+} -+ -+static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ -+ lockdep_assert_held(&kctx->jctx.lock); -+ list_add_tail(&katom->jit_node, &kctx->jit_atoms_head); -+ -+ return 0; -+} -+ -+static void kbase_jit_free_process(struct kbase_jd_atom *katom) -+{ -+ struct kbase_context *kctx = katom->kctx; -+ u8 id = kbase_jit_free_get_id(katom); -+ -+ /* -+ * If the ID is zero or it is not in use yet then fail the job. -+ */ -+ if ((id == 0) || (kctx->jit_alloc[id] == NULL)) { -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ return; -+ } -+ -+ /* -+ * If the ID is valid but the allocation request failed still succeed -+ * this soft job but don't try and free the allocation. -+ */ -+ if (kctx->jit_alloc[id] != (struct kbase_va_region *) -1) -+ kbase_jit_free(kctx, kctx->jit_alloc[id]); -+ -+ kctx->jit_alloc[id] = NULL; -+} -+ -+static void kbasep_jit_free_finish_worker(struct work_struct *work) -+{ -+ struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, -+ work); -+ struct kbase_context *kctx = katom->kctx; -+ int resched; -+ -+ mutex_lock(&kctx->jctx.lock); -+ kbase_finish_soft_job(katom); -+ resched = jd_done_nolock(katom, NULL); -+ mutex_unlock(&kctx->jctx.lock); -+ -+ if (resched) -+ kbase_js_sched_all(kctx->kbdev); -+} -+ -+static void kbase_jit_free_finish(struct kbase_jd_atom *katom) -+{ -+ struct list_head *i, *tmp; -+ struct kbase_context *kctx = katom->kctx; -+ -+ lockdep_assert_held(&kctx->jctx.lock); -+ /* Remove this atom from the kctx->jit_atoms_head list */ -+ list_del(&katom->jit_node); -+ -+ list_for_each_safe(i, tmp, &kctx->jit_pending_alloc) { -+ struct kbase_jd_atom *pending_atom = list_entry(i, -+ struct kbase_jd_atom, queue); -+ if (kbase_jit_allocate_process(pending_atom) == 0) { -+ /* Atom has completed */ -+ INIT_WORK(&pending_atom->work, -+ kbasep_jit_free_finish_worker); -+ queue_work(kctx->jctx.job_done_wq, &pending_atom->work); -+ } -+ } -+} -+ -+static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) -+{ -+ __user struct base_external_resource_list *user_ext_res; -+ struct base_external_resource_list *ext_res; -+ u64 count = 0; -+ size_t copy_size; -+ int ret; -+ -+ user_ext_res = (__user struct base_external_resource_list *) -+ (uintptr_t) katom->jc; -+ -+ /* Fail the job if there is no info structure */ -+ if (!user_ext_res) { -+ ret = -EINVAL; -+ goto fail; -+ } -+ -+ if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { -+ ret = -EINVAL; -+ goto fail; -+ } -+ -+ /* Is the number of external resources in range? */ -+ if (!count || count > BASE_EXT_RES_COUNT_MAX) { -+ ret = -EINVAL; -+ goto fail; -+ } -+ -+ /* Copy the information for safe access and future storage */ -+ copy_size = sizeof(*ext_res); -+ copy_size += sizeof(struct base_external_resource) * (count - 1); -+ ext_res = kzalloc(copy_size, GFP_KERNEL); -+ if (!ext_res) { -+ ret = -ENOMEM; -+ goto fail; -+ } -+ -+ if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { -+ ret = -EINVAL; -+ goto free_info; -+ } -+ -+ /* -+ * Overwrite the count with the first value incase it was changed -+ * after the fact. -+ */ -+ ext_res->count = count; -+ -+ /* -+ * Replace the user pointer with our kernel allocated -+ * ext_res structure. -+ */ -+ katom->jc = (u64)(uintptr_t) ext_res; -+ -+ return 0; -+ -+free_info: -+ kfree(ext_res); -+fail: -+ return ret; -+} -+ -+static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) -+{ -+ struct base_external_resource_list *ext_res; -+ int i; -+ bool failed = false; -+ -+ ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; -+ if (!ext_res) -+ goto failed_jc; -+ -+ kbase_gpu_vm_lock(katom->kctx); -+ -+ for (i = 0; i < ext_res->count; i++) { -+ u64 gpu_addr; -+ -+ gpu_addr = ext_res->ext_res[i].ext_resource & -+ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; -+ if (map) { -+ if (!kbase_sticky_resource_acquire(katom->kctx, -+ gpu_addr)) -+ goto failed_loop; -+ } else -+ if (!kbase_sticky_resource_release(katom->kctx, NULL, -+ gpu_addr)) -+ failed = true; -+ } -+ -+ /* -+ * In the case of unmap we continue unmapping other resources in the -+ * case of failure but will always report failure if _any_ unmap -+ * request fails. -+ */ -+ if (failed) -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ else -+ katom->event_code = BASE_JD_EVENT_DONE; -+ -+ kbase_gpu_vm_unlock(katom->kctx); -+ -+ return; -+ -+failed_loop: -+ while (--i > 0) { -+ u64 gpu_addr; -+ -+ gpu_addr = ext_res->ext_res[i].ext_resource & -+ ~BASE_EXT_RES_ACCESS_EXCLUSIVE; -+ -+ kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr); -+ } -+ -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ kbase_gpu_vm_unlock(katom->kctx); -+ -+failed_jc: -+ return; -+} -+ -+static void kbase_ext_res_finish(struct kbase_jd_atom *katom) -+{ -+ struct base_external_resource_list *ext_res; -+ -+ ext_res = (struct base_external_resource_list *) (uintptr_t) katom->jc; -+ /* Free the info structure */ -+ kfree(ext_res); -+} -+ -+int kbase_process_soft_job(struct kbase_jd_atom *katom) -+{ -+ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: -+ return kbase_dump_cpu_gpu_time(katom); -+ -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: -+ katom->event_code = kbase_sync_fence_out_trigger(katom, -+ katom->event_code == BASE_JD_EVENT_DONE ? -+ 0 : -EFAULT); -+ break; -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ { -+ int ret = kbase_sync_fence_in_wait(katom); -+ -+ if (ret == 1) { -+#ifdef CONFIG_MALI_FENCE_DEBUG -+ kbasep_add_waiting_with_timeout(katom); -+#else -+ kbasep_add_waiting_soft_job(katom); -+#endif -+ } -+ return ret; -+ } -+#endif -+ -+ case BASE_JD_REQ_SOFT_REPLAY: -+ return kbase_replay_process(katom); -+ case BASE_JD_REQ_SOFT_EVENT_WAIT: -+ return kbasep_soft_event_wait(katom); -+ case BASE_JD_REQ_SOFT_EVENT_SET: -+ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); -+ break; -+ case BASE_JD_REQ_SOFT_EVENT_RESET: -+ kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); -+ break; -+ case BASE_JD_REQ_SOFT_DEBUG_COPY: -+ { -+ int res = kbase_debug_copy(katom); -+ -+ if (res) -+ katom->event_code = BASE_JD_EVENT_JOB_INVALID; -+ break; -+ } -+ case BASE_JD_REQ_SOFT_JIT_ALLOC: -+ return kbase_jit_allocate_process(katom); -+ case BASE_JD_REQ_SOFT_JIT_FREE: -+ kbase_jit_free_process(katom); -+ break; -+ case BASE_JD_REQ_SOFT_EXT_RES_MAP: -+ kbase_ext_res_process(katom, true); -+ break; -+ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: -+ kbase_ext_res_process(katom, false); -+ break; -+ } -+ -+ /* Atom is complete */ -+ return 0; -+} -+ -+void kbase_cancel_soft_job(struct kbase_jd_atom *katom) -+{ -+ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ kbase_sync_fence_in_cancel_wait(katom); -+ break; -+#endif -+ case BASE_JD_REQ_SOFT_EVENT_WAIT: -+ kbasep_soft_event_cancel_job(katom); -+ break; -+ default: -+ /* This soft-job doesn't support cancellation! */ -+ KBASE_DEBUG_ASSERT(0); -+ } -+} -+ -+int kbase_prepare_soft_job(struct kbase_jd_atom *katom) -+{ -+ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: -+ { -+ if (0 != (katom->jc & KBASE_CACHE_ALIGNMENT_MASK)) -+ return -EINVAL; -+ } -+ break; -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: -+ { -+ struct base_fence fence; -+ int fd; -+ -+ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) -+ return -EINVAL; -+ -+ fd = kbase_sync_fence_out_create(katom, -+ fence.basep.stream_fd); -+ if (fd < 0) -+ return -EINVAL; -+ -+ fence.basep.fd = fd; -+ if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { -+ kbase_sync_fence_out_remove(katom); -+ kbase_sync_fence_close_fd(fd); -+ fence.basep.fd = -EINVAL; -+ return -EINVAL; -+ } -+ } -+ break; -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ { -+ struct base_fence fence; -+ int ret; -+ -+ if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) -+ return -EINVAL; -+ -+ /* Get a reference to the fence object */ -+ ret = kbase_sync_fence_in_from_fd(katom, -+ fence.basep.fd); -+ if (ret < 0) -+ return ret; -+ -+#ifdef CONFIG_MALI_DMA_FENCE -+ /* -+ * Set KCTX_NO_IMPLICIT_FENCE in the context the first -+ * time a soft fence wait job is observed. This will -+ * prevent the implicit dma-buf fence to conflict with -+ * the Android native sync fences. -+ */ -+ if (!kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC)) -+ kbase_ctx_flag_set(katom->kctx, KCTX_NO_IMPLICIT_SYNC); -+#endif /* CONFIG_MALI_DMA_FENCE */ -+ } -+ break; -+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+ case BASE_JD_REQ_SOFT_JIT_ALLOC: -+ return kbase_jit_allocate_prepare(katom); -+ case BASE_JD_REQ_SOFT_REPLAY: -+ break; -+ case BASE_JD_REQ_SOFT_JIT_FREE: -+ return kbase_jit_free_prepare(katom); -+ case BASE_JD_REQ_SOFT_EVENT_WAIT: -+ case BASE_JD_REQ_SOFT_EVENT_SET: -+ case BASE_JD_REQ_SOFT_EVENT_RESET: -+ if (katom->jc == 0) -+ return -EINVAL; -+ break; -+ case BASE_JD_REQ_SOFT_DEBUG_COPY: -+ return kbase_debug_copy_prepare(katom); -+ case BASE_JD_REQ_SOFT_EXT_RES_MAP: -+ return kbase_ext_res_prepare(katom); -+ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: -+ return kbase_ext_res_prepare(katom); -+ default: -+ /* Unsupported soft-job */ -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+void kbase_finish_soft_job(struct kbase_jd_atom *katom) -+{ -+ switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -+ case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: -+ /* Nothing to do */ -+ break; -+#if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) -+ case BASE_JD_REQ_SOFT_FENCE_TRIGGER: -+ /* If fence has not yet been signaled, do it now */ -+ kbase_sync_fence_out_trigger(katom, katom->event_code == -+ BASE_JD_EVENT_DONE ? 0 : -EFAULT); -+ break; -+ case BASE_JD_REQ_SOFT_FENCE_WAIT: -+ /* Release katom's reference to fence object */ -+ kbase_sync_fence_in_remove(katom); -+ break; -+#endif /* CONFIG_SYNC || CONFIG_SYNC_FILE */ -+ case BASE_JD_REQ_SOFT_DEBUG_COPY: -+ kbase_debug_copy_finish(katom); -+ break; -+ case BASE_JD_REQ_SOFT_JIT_ALLOC: -+ kbase_jit_allocate_finish(katom); -+ break; -+ case BASE_JD_REQ_SOFT_EXT_RES_MAP: -+ kbase_ext_res_finish(katom); -+ break; -+ case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: -+ kbase_ext_res_finish(katom); -+ break; -+ case BASE_JD_REQ_SOFT_JIT_FREE: -+ kbase_jit_free_finish(katom); -+ break; -+ } -+} -+ -+void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) -+{ -+ LIST_HEAD(local_suspended_soft_jobs); -+ struct kbase_jd_atom *tmp_iter; -+ struct kbase_jd_atom *katom_iter; -+ struct kbasep_js_device_data *js_devdata; -+ bool resched = false; -+ -+ KBASE_DEBUG_ASSERT(kbdev); -+ -+ js_devdata = &kbdev->js_data; -+ -+ /* Move out the entire list */ -+ mutex_lock(&js_devdata->runpool_mutex); -+ list_splice_init(&js_devdata->suspended_soft_jobs_list, -+ &local_suspended_soft_jobs); -+ mutex_unlock(&js_devdata->runpool_mutex); -+ -+ /* -+ * Each atom must be detached from the list and ran separately - -+ * it could be re-added to the old list, but this is unlikely -+ */ -+ list_for_each_entry_safe(katom_iter, tmp_iter, -+ &local_suspended_soft_jobs, dep_item[1]) { -+ struct kbase_context *kctx = katom_iter->kctx; -+ -+ mutex_lock(&kctx->jctx.lock); -+ -+ /* Remove from the global list */ -+ list_del(&katom_iter->dep_item[1]); -+ /* Remove from the context's list of waiting soft jobs */ -+ kbasep_remove_waiting_soft_job(katom_iter); -+ -+ if (kbase_process_soft_job(katom_iter) == 0) { -+ kbase_finish_soft_job(katom_iter); -+ resched |= jd_done_nolock(katom_iter, NULL); -+ } else { -+ KBASE_DEBUG_ASSERT((katom_iter->core_req & -+ BASE_JD_REQ_SOFT_JOB_TYPE) -+ != BASE_JD_REQ_SOFT_REPLAY); -+ } -+ -+ mutex_unlock(&kctx->jctx.lock); -+ } -+ -+ if (resched) -+ kbase_js_sched_all(kbdev); -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_strings.c b/drivers/gpu/arm/midgard/mali_kbase_strings.c -new file mode 100755 -index 000000000..c98762cec ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_strings.c -@@ -0,0 +1,23 @@ -+ /* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+#include "mali_kbase_strings.h" -+ -+#define KBASE_DRV_NAME "mali" -+#define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" -+ -+const char kbase_drv_name[] = KBASE_DRV_NAME; -+const char kbase_timeline_name[] = KBASE_TIMELINE_NAME; -diff --git a/drivers/gpu/arm/midgard/mali_kbase_strings.h b/drivers/gpu/arm/midgard/mali_kbase_strings.h -new file mode 100755 -index 000000000..41b8fdbec ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_strings.h -@@ -0,0 +1,19 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+extern const char kbase_drv_name[]; -+extern const char kbase_timeline_name[]; -diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync.h b/drivers/gpu/arm/midgard/mali_kbase_sync.h -new file mode 100755 -index 000000000..33b580595 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_sync.h -@@ -0,0 +1,203 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * @file mali_kbase_sync.h -+ * -+ * This file contains our internal "API" for explicit fences. -+ * It hides the implementation details of the actual explicit fence mechanism -+ * used (Android fences or sync file with DMA fences). -+ */ -+ -+#ifndef MALI_KBASE_SYNC_H -+#define MALI_KBASE_SYNC_H -+ -+#include -+#ifdef CONFIG_SYNC -+#include -+#endif -+#ifdef CONFIG_SYNC_FILE -+#include "mali_kbase_fence_defs.h" -+#include -+#endif -+ -+#include "mali_kbase.h" -+ -+/** -+ * struct kbase_sync_fence_info - Information about a fence -+ * @fence: Pointer to fence (type is void*, as underlaying struct can differ) -+ * @name: The name given to this fence when it was created -+ * @status: < 0 means error, 0 means active, 1 means signaled -+ * -+ * Use kbase_sync_fence_in_info_get() or kbase_sync_fence_out_info_get() -+ * to get the information. -+ */ -+struct kbase_sync_fence_info { -+ void *fence; -+ char name[32]; -+ int status; -+}; -+ -+/** -+ * kbase_sync_fence_stream_create() - Create a stream object -+ * @name: Name of stream (only used to ease debugging/visualization) -+ * @out_fd: A file descriptor representing the created stream object -+ * -+ * Can map down to a timeline implementation in some implementations. -+ * Exposed as a file descriptor. -+ * Life-time controlled via the file descriptor: -+ * - dup to add a ref -+ * - close to remove a ref -+ * -+ * return: 0 on success, < 0 on error -+ */ -+int kbase_sync_fence_stream_create(const char *name, int *const out_fd); -+ -+/** -+ * kbase_sync_fence_out_create Create an explicit output fence to specified atom -+ * @katom: Atom to assign the new explicit fence to -+ * @stream_fd: File descriptor for stream object to create fence on -+ * -+ * return: Valid file descriptor to fence or < 0 on error -+ */ -+int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); -+ -+/** -+ * kbase_sync_fence_in_from_fd() Assigns an existing fence to specified atom -+ * @katom: Atom to assign the existing explicit fence to -+ * @fd: File descriptor to an existing fence -+ * -+ * Assigns an explicit input fence to atom. -+ * This can later be waited for by calling @kbase_sync_fence_in_wait -+ * -+ * return: 0 on success, < 0 on error -+ */ -+int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); -+ -+/** -+ * kbase_sync_fence_validate() - Validate a fd to be a valid fence -+ * @fd: File descriptor to check -+ * -+ * This function is only usable to catch unintentional user errors early, -+ * it does not stop malicious code changing the fd after this function returns. -+ * -+ * return 0: if fd is for a valid fence, < 0 if invalid -+ */ -+int kbase_sync_fence_validate(int fd); -+ -+/** -+ * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom -+ * @katom: Atom with an explicit fence to signal -+ * @result: < 0 means signal with error, 0 >= indicates success -+ * -+ * Signal output fence attached on katom and remove the fence from the atom. -+ * -+ * return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE -+ */ -+enum base_jd_event_code -+kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); -+ -+/** -+ * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled -+ * @katom: Atom with explicit fence to wait for -+ * -+ * If the fence is already signaled, then 0 is returned, and the caller must -+ * continue processing of the katom. -+ * -+ * If the fence isn't already signaled, then this kbase_sync framework will -+ * take responsibility to continue the processing once the fence is signaled. -+ * -+ * return: 0 if already signaled, otherwise 1 -+ */ -+int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits -+ * @katom: Atom to cancel wait for -+ * -+ * This function is fully responsible for continuing processing of this atom -+ * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) -+ */ -+void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_sync_fence_in_remove() - Remove the input fence from the katom -+ * @katom: Atom to remove explicit input fence for -+ * -+ * This will also release the corresponding reference. -+ */ -+void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_sync_fence_out_remove() - Remove the output fence from the katom -+ * @katom: Atom to remove explicit output fence for -+ * -+ * This will also release the corresponding reference. -+ */ -+void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); -+ -+/** -+ * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence -+ * @fd: File descriptor to close -+ */ -+static inline void kbase_sync_fence_close_fd(int fd) -+{ -+ ksys_close(fd); -+} -+ -+/** -+ * kbase_sync_fence_in_info_get() - Retrieves information about input fence -+ * @katom: Atom to get fence information from -+ * @info: Struct to be filled with fence information -+ * -+ * return: 0 on success, < 0 on error -+ */ -+int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, -+ struct kbase_sync_fence_info *info); -+ -+/** -+ * kbase_sync_fence_out_info_get() - Retrieves information about output fence -+ * @katom: Atom to get fence information from -+ * @info: Struct to be filled with fence information -+ * -+ * return: 0 on success, < 0 on error -+ */ -+int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, -+ struct kbase_sync_fence_info *info); -+ -+/** -+ * kbase_sync_status_string() - Get string matching @status -+ * @status: Value of fence status. -+ * -+ * return: Pointer to string describing @status. -+ */ -+const char *kbase_sync_status_string(int status); -+ -+/* -+ * Internal worker used to continue processing of atom. -+ */ -+void kbase_sync_fence_wait_worker(struct work_struct *data); -+ -+#ifdef CONFIG_MALI_FENCE_DEBUG -+/** -+ * kbase_sync_fence_in_dump() Trigger a debug dump of atoms input fence state -+ * @katom: Atom to trigger fence debug dump for -+ */ -+void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); -+#endif -+ -+#endif /* MALI_KBASE_SYNC_H */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_android.c b/drivers/gpu/arm/midgard/mali_kbase_sync_android.c -new file mode 100755 -index 000000000..d7349dcae ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_sync_android.c -@@ -0,0 +1,537 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Code for supporting explicit Android fences (CONFIG_SYNC) -+ * Known to be good for kernels 4.5 and earlier. -+ * Replaced with CONFIG_SYNC_FILE for 4.9 and later kernels -+ * (see mali_kbase_sync_file.c) -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "sync.h" -+#include -+#include -+ -+struct mali_sync_timeline { -+ struct sync_timeline timeline; -+ atomic_t counter; -+ atomic_t signaled; -+}; -+ -+struct mali_sync_pt { -+ struct sync_pt pt; -+ int order; -+ int result; -+}; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+/* For backwards compatibility with kernels before 3.17. After 3.17 -+ * sync_pt_parent is included in the kernel. */ -+static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) -+{ -+ return pt->parent; -+} -+#endif -+ -+static struct mali_sync_timeline *to_mali_sync_timeline( -+ struct sync_timeline *timeline) -+{ -+ return container_of(timeline, struct mali_sync_timeline, timeline); -+} -+ -+static struct mali_sync_pt *to_mali_sync_pt(struct sync_pt *pt) -+{ -+ return container_of(pt, struct mali_sync_pt, pt); -+} -+ -+static struct sync_pt *timeline_dup(struct sync_pt *pt) -+{ -+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); -+ struct mali_sync_pt *new_mpt; -+ struct sync_pt *new_pt = sync_pt_create(sync_pt_parent(pt), -+ sizeof(struct mali_sync_pt)); -+ -+ if (!new_pt) -+ return NULL; -+ -+ new_mpt = to_mali_sync_pt(new_pt); -+ new_mpt->order = mpt->order; -+ new_mpt->result = mpt->result; -+ -+ return new_pt; -+} -+ -+static int timeline_has_signaled(struct sync_pt *pt) -+{ -+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); -+ struct mali_sync_timeline *mtl = to_mali_sync_timeline( -+ sync_pt_parent(pt)); -+ int result = mpt->result; -+ -+ int diff = atomic_read(&mtl->signaled) - mpt->order; -+ -+ if (diff >= 0) -+ return (result < 0) ? result : 1; -+ -+ return 0; -+} -+ -+static int timeline_compare(struct sync_pt *a, struct sync_pt *b) -+{ -+ struct mali_sync_pt *ma = container_of(a, struct mali_sync_pt, pt); -+ struct mali_sync_pt *mb = container_of(b, struct mali_sync_pt, pt); -+ -+ int diff = ma->order - mb->order; -+ -+ if (diff == 0) -+ return 0; -+ -+ return (diff < 0) ? -1 : 1; -+} -+ -+static void timeline_value_str(struct sync_timeline *timeline, char *str, -+ int size) -+{ -+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(timeline); -+ -+ snprintf(str, size, "%d", atomic_read(&mtl->signaled)); -+} -+ -+static void pt_value_str(struct sync_pt *pt, char *str, int size) -+{ -+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); -+ -+ snprintf(str, size, "%d(%d)", mpt->order, mpt->result); -+} -+ -+static struct sync_timeline_ops mali_timeline_ops = { -+ .driver_name = "Mali", -+ .dup = timeline_dup, -+ .has_signaled = timeline_has_signaled, -+ .compare = timeline_compare, -+ .timeline_value_str = timeline_value_str, -+ .pt_value_str = pt_value_str, -+}; -+ -+/* Allocates a timeline for Mali -+ * -+ * One timeline should be allocated per API context. -+ */ -+static struct sync_timeline *mali_sync_timeline_alloc(const char *name) -+{ -+ struct sync_timeline *tl; -+ struct mali_sync_timeline *mtl; -+ -+ tl = sync_timeline_create(&mali_timeline_ops, -+ sizeof(struct mali_sync_timeline), name); -+ if (!tl) -+ return NULL; -+ -+ /* Set the counter in our private struct */ -+ mtl = to_mali_sync_timeline(tl); -+ atomic_set(&mtl->counter, 0); -+ atomic_set(&mtl->signaled, 0); -+ -+ return tl; -+} -+ -+static int kbase_stream_close(struct inode *inode, struct file *file) -+{ -+ struct sync_timeline *tl; -+ -+ tl = (struct sync_timeline *)file->private_data; -+ sync_timeline_destroy(tl); -+ return 0; -+} -+ -+static const struct file_operations stream_fops = { -+ .owner = THIS_MODULE, -+ .release = kbase_stream_close, -+}; -+ -+int kbase_sync_fence_stream_create(const char *name, int *const out_fd) -+{ -+ struct sync_timeline *tl; -+ -+ if (!out_fd) -+ return -EINVAL; -+ -+ tl = mali_sync_timeline_alloc(name); -+ if (!tl) -+ return -EINVAL; -+ -+ *out_fd = anon_inode_getfd(name, &stream_fops, tl, O_RDONLY|O_CLOEXEC); -+ -+ if (*out_fd < 0) { -+ sync_timeline_destroy(tl); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+/* Allocates a sync point within the timeline. -+ * -+ * The timeline must be the one allocated by kbase_sync_timeline_alloc -+ * -+ * Sync points must be triggered in *exactly* the same order as they are -+ * allocated. -+ */ -+static struct sync_pt *kbase_sync_pt_alloc(struct sync_timeline *parent) -+{ -+ struct sync_pt *pt = sync_pt_create(parent, -+ sizeof(struct mali_sync_pt)); -+ struct mali_sync_timeline *mtl = to_mali_sync_timeline(parent); -+ struct mali_sync_pt *mpt; -+ -+ if (!pt) -+ return NULL; -+ -+ mpt = to_mali_sync_pt(pt); -+ mpt->order = atomic_inc_return(&mtl->counter); -+ mpt->result = 0; -+ -+ return pt; -+} -+ -+int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) -+{ -+ struct sync_timeline *tl; -+ struct sync_pt *pt; -+ struct sync_fence *fence; -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) -+ struct files_struct *files; -+ struct fdtable *fdt; -+#endif -+ int fd; -+ struct file *tl_file; -+ -+ tl_file = fget(tl_fd); -+ if (tl_file == NULL) -+ return -EBADF; -+ -+ if (tl_file->f_op != &stream_fops) { -+ fd = -EBADF; -+ goto out; -+ } -+ -+ tl = tl_file->private_data; -+ -+ pt = kbase_sync_pt_alloc(tl); -+ if (!pt) { -+ fd = -EFAULT; -+ goto out; -+ } -+ -+ fence = sync_fence_create("mali_fence", pt); -+ if (!fence) { -+ sync_pt_free(pt); -+ fd = -EFAULT; -+ goto out; -+ } -+ -+ /* from here the fence owns the sync_pt */ -+ -+ /* create a fd representing the fence */ -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) -+ fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); -+ if (fd < 0) { -+ sync_fence_put(fence); -+ goto out; -+ } -+#else -+ fd = get_unused_fd(); -+ if (fd < 0) { -+ sync_fence_put(fence); -+ goto out; -+ } -+ -+ files = current->files; -+ spin_lock(&files->file_lock); -+ fdt = files_fdtable(files); -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) -+ __set_close_on_exec(fd, fdt); -+#else -+ FD_SET(fd, fdt->close_on_exec); -+#endif -+ spin_unlock(&files->file_lock); -+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ -+ -+ /* bind fence to the new fd */ -+ sync_fence_install(fence, fd); -+ -+ katom->fence = sync_fence_fdget(fd); -+ if (katom->fence == NULL) { -+ /* The only way the fence can be NULL is if userspace closed it -+ * for us, so we don't need to clear it up */ -+ fd = -EINVAL; -+ goto out; -+ } -+ -+out: -+ fput(tl_file); -+ -+ return fd; -+} -+ -+int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) -+{ -+ katom->fence = sync_fence_fdget(fd); -+ return katom->fence ? 0 : -ENOENT; -+} -+ -+int kbase_sync_fence_validate(int fd) -+{ -+ struct sync_fence *fence; -+ -+ fence = sync_fence_fdget(fd); -+ if (!fence) -+ return -EINVAL; -+ -+ sync_fence_put(fence); -+ return 0; -+} -+ -+/* Returns true if the specified timeline is allocated by Mali */ -+static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) -+{ -+ return timeline->ops == &mali_timeline_ops; -+} -+ -+/* Signals a particular sync point -+ * -+ * Sync points must be triggered in *exactly* the same order as they are -+ * allocated. -+ * -+ * If they are signaled in the wrong order then a message will be printed in -+ * debug builds and otherwise attempts to signal order sync_pts will be ignored. -+ * -+ * result can be negative to indicate error, any other value is interpreted as -+ * success. -+ */ -+static void kbase_sync_signal_pt(struct sync_pt *pt, int result) -+{ -+ struct mali_sync_pt *mpt = to_mali_sync_pt(pt); -+ struct mali_sync_timeline *mtl = to_mali_sync_timeline( -+ sync_pt_parent(pt)); -+ int signaled; -+ int diff; -+ -+ mpt->result = result; -+ -+ do { -+ signaled = atomic_read(&mtl->signaled); -+ -+ diff = signaled - mpt->order; -+ -+ if (diff > 0) { -+ /* The timeline is already at or ahead of this point. -+ * This should not happen unless userspace has been -+ * signaling fences out of order, so warn but don't -+ * violate the sync_pt API. -+ * The warning is only in debug builds to prevent -+ * a malicious user being able to spam dmesg. -+ */ -+#ifdef CONFIG_MALI_DEBUG -+ pr_err("Fences were triggered in a different order to allocation!"); -+#endif /* CONFIG_MALI_DEBUG */ -+ return; -+ } -+ } while (atomic_cmpxchg(&mtl->signaled, -+ signaled, mpt->order) != signaled); -+} -+ -+enum base_jd_event_code -+kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) -+{ -+ struct sync_pt *pt; -+ struct sync_timeline *timeline; -+ -+ if (!katom->fence) -+ return BASE_JD_EVENT_JOB_CANCELLED; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+ if (!list_is_singular(&katom->fence->pt_list_head)) { -+#else -+ if (katom->fence->num_fences != 1) { -+#endif -+ /* Not exactly one item in the list - so it didn't (directly) -+ * come from us */ -+ return BASE_JD_EVENT_JOB_CANCELLED; -+ } -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+ pt = list_first_entry(&katom->fence->pt_list_head, -+ struct sync_pt, pt_list); -+#else -+ pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); -+#endif -+ timeline = sync_pt_parent(pt); -+ -+ if (!kbase_sync_timeline_is_ours(timeline)) { -+ /* Fence has a sync_pt which isn't ours! */ -+ return BASE_JD_EVENT_JOB_CANCELLED; -+ } -+ -+ kbase_sync_signal_pt(pt, result); -+ -+ sync_timeline_signal(timeline); -+ -+ kbase_sync_fence_out_remove(katom); -+ -+ return (result < 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; -+} -+ -+static inline int kbase_fence_get_status(struct sync_fence *fence) -+{ -+ if (!fence) -+ return -ENOENT; -+ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -+ return fence->status; -+#else -+ return atomic_read(&fence->status); -+#endif -+} -+ -+static void kbase_fence_wait_callback(struct sync_fence *fence, -+ struct sync_fence_waiter *waiter) -+{ -+ struct kbase_jd_atom *katom = container_of(waiter, -+ struct kbase_jd_atom, sync_waiter); -+ struct kbase_context *kctx = katom->kctx; -+ -+ /* Propagate the fence status to the atom. -+ * If negative then cancel this atom and its dependencies. -+ */ -+ if (kbase_fence_get_status(fence) < 0) -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ /* To prevent a potential deadlock we schedule the work onto the -+ * job_done_wq workqueue -+ * -+ * The issue is that we may signal the timeline while holding -+ * kctx->jctx.lock and the callbacks are run synchronously from -+ * sync_timeline_signal. So we simply defer the work. -+ */ -+ -+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); -+ queue_work(kctx->jctx.job_done_wq, &katom->work); -+} -+ -+int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) -+{ -+ int ret; -+ -+ sync_fence_waiter_init(&katom->sync_waiter, kbase_fence_wait_callback); -+ -+ ret = sync_fence_wait_async(katom->fence, &katom->sync_waiter); -+ -+ if (ret == 1) { -+ /* Already signaled */ -+ return 0; -+ } -+ -+ if (ret < 0) { -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ /* We should cause the dependent jobs in the bag to be failed, -+ * to do this we schedule the work queue to complete this job */ -+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); -+ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); -+ } -+ -+ return 1; -+} -+ -+void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) -+{ -+ if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { -+ /* The wait wasn't cancelled - leave the cleanup for -+ * kbase_fence_wait_callback */ -+ return; -+ } -+ -+ /* Wait was cancelled - zap the atoms */ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ kbasep_remove_waiting_soft_job(katom); -+ kbase_finish_soft_job(katom); -+ -+ if (jd_done_nolock(katom, NULL)) -+ kbase_js_sched_all(katom->kctx->kbdev); -+} -+ -+void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) -+{ -+ if (katom->fence) { -+ sync_fence_put(katom->fence); -+ katom->fence = NULL; -+ } -+} -+ -+void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) -+{ -+ if (katom->fence) { -+ sync_fence_put(katom->fence); -+ katom->fence = NULL; -+ } -+} -+ -+int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, -+ struct kbase_sync_fence_info *info) -+{ -+ if (!katom->fence) -+ return -ENOENT; -+ -+ info->fence = katom->fence; -+ info->status = kbase_fence_get_status(katom->fence); -+ strlcpy(info->name, katom->fence->name, sizeof(info->name)); -+ -+ return 0; -+} -+ -+int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, -+ struct kbase_sync_fence_info *info) -+{ -+ if (!katom->fence) -+ return -ENOENT; -+ -+ info->fence = katom->fence; -+ info->status = kbase_fence_get_status(katom->fence); -+ strlcpy(info->name, katom->fence->name, sizeof(info->name)); -+ -+ return 0; -+} -+ -+#ifdef CONFIG_MALI_FENCE_DEBUG -+void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) -+{ -+ /* Dump out the full state of all the Android sync fences. -+ * The function sync_dump() isn't exported to modules, so force -+ * sync_fence_wait() to time out to trigger sync_dump(). -+ */ -+ if (katom->fence) -+ sync_fence_wait(katom->fence, 1); -+} -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_common.c b/drivers/gpu/arm/midgard/mali_kbase_sync_common.c -new file mode 100755 -index 000000000..457def296 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_sync_common.c -@@ -0,0 +1,43 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * @file mali_kbase_sync_common.c -+ * -+ * Common code for our explicit fence functionality -+ */ -+ -+#include -+#include "mali_kbase.h" -+ -+void kbase_sync_fence_wait_worker(struct work_struct *data) -+{ -+ struct kbase_jd_atom *katom; -+ -+ katom = container_of(data, struct kbase_jd_atom, work); -+ kbase_soft_event_wait_callback(katom); -+} -+ -+const char *kbase_sync_status_string(int status) -+{ -+ if (status == 0) -+ return "signaled"; -+ else if (status > 0) -+ return "active"; -+ else -+ return "error"; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_sync_file.c b/drivers/gpu/arm/midgard/mali_kbase_sync_file.c -new file mode 100755 -index 000000000..60b5d74db ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_sync_file.c -@@ -0,0 +1,359 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* -+ * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) -+ * Introduced in kernel 4.9. -+ * Android explicit fences (CONFIG_SYNC) can be used for older kernels -+ * (see mali_kbase_sync_android.c) -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include "mali_kbase_fence_defs.h" -+#include "mali_kbase_sync.h" -+#include "mali_kbase_fence.h" -+#include "mali_kbase.h" -+ -+static const struct file_operations stream_fops = { -+ .owner = THIS_MODULE -+}; -+ -+int kbase_sync_fence_stream_create(const char *name, int *const out_fd) -+{ -+ if (!out_fd) -+ return -EINVAL; -+ -+ *out_fd = anon_inode_getfd(name, &stream_fops, NULL, -+ O_RDONLY | O_CLOEXEC); -+ if (*out_fd < 0) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ struct sync_file *sync_file; -+ int fd; -+ -+ fence = kbase_fence_out_new(katom); -+ if (!fence) -+ return -ENOMEM; -+ -+#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) -+ /* Take an extra reference to the fence on behalf of the sync_file. -+ * This is only needed on older kernels where sync_file_create() -+ * does not take its own reference. This was changed in v4.9.68, -+ * where sync_file_create() now takes its own reference. -+ */ -+ dma_fence_get(fence); -+#endif -+ -+ /* create a sync_file fd representing the fence */ -+ sync_file = sync_file_create(fence); -+ if (!sync_file) { -+ dma_fence_put(fence); -+ kbase_fence_out_remove(katom); -+ return -ENOMEM; -+ } -+ -+ fd = get_unused_fd_flags(O_CLOEXEC); -+ if (fd < 0) { -+ fput(sync_file->file); -+ kbase_fence_out_remove(katom); -+ return fd; -+ } -+ -+ fd_install(fd, sync_file->file); -+ -+ return fd; -+} -+ -+int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence = sync_file_get_fence(fd); -+#else -+ struct dma_fence *fence = sync_file_get_fence(fd); -+#endif -+ -+ if (!fence) -+ return -ENOENT; -+ -+ kbase_fence_fence_in_set(katom, fence); -+ -+ return 0; -+} -+ -+int kbase_sync_fence_validate(int fd) -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence = sync_file_get_fence(fd); -+#else -+ struct dma_fence *fence = sync_file_get_fence(fd); -+#endif -+ -+ if (!fence) -+ return -EINVAL; -+ -+ dma_fence_put(fence); -+ -+ return 0; /* valid */ -+} -+ -+enum base_jd_event_code -+kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) -+{ -+ int res; -+ -+ if (!kbase_fence_out_is_ours(katom)) { -+ /* Not our fence */ -+ return BASE_JD_EVENT_JOB_CANCELLED; -+ } -+ -+ res = kbase_fence_out_signal(katom, result); -+ if (unlikely(res < 0)) { -+ dev_warn(katom->kctx->kbdev->dev, -+ "fence_signal() failed with %d\n", res); -+ } -+ -+ kbase_sync_fence_out_remove(katom); -+ -+ return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; -+} -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+static void kbase_fence_wait_callback(struct fence *fence, -+ struct fence_cb *cb) -+#else -+static void kbase_fence_wait_callback(struct dma_fence *fence, -+ struct dma_fence_cb *cb) -+#endif -+{ -+ struct kbase_fence_cb *kcb = container_of(cb, -+ struct kbase_fence_cb, -+ fence_cb); -+ struct kbase_jd_atom *katom = kcb->katom; -+ struct kbase_context *kctx = katom->kctx; -+ -+ /* Cancel atom if fence is erroneous */ -+#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ -+ (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ -+ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) -+ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error) -+#else -+ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) -+#endif -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ if (kbase_fence_dep_count_dec_and_test(katom)) { -+ /* We take responsibility of handling this */ -+ kbase_fence_dep_count_set(katom, -1); -+ -+ /* To prevent a potential deadlock we schedule the work onto the -+ * job_done_wq workqueue -+ * -+ * The issue is that we may signal the timeline while holding -+ * kctx->jctx.lock and the callbacks are run synchronously from -+ * sync_timeline_signal. So we simply defer the work. -+ */ -+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); -+ queue_work(kctx->jctx.job_done_wq, &katom->work); -+ } -+} -+ -+int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) -+{ -+ int err; -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ -+ fence = kbase_fence_in_get(katom); -+ if (!fence) -+ return 0; /* no input fence to wait for, good to go! */ -+ -+ kbase_fence_dep_count_set(katom, 1); -+ -+ err = kbase_fence_add_callback(katom, fence, kbase_fence_wait_callback); -+ -+ kbase_fence_put(fence); -+ -+ if (likely(!err)) { -+ /* Test if the callbacks are already triggered */ -+ if (kbase_fence_dep_count_dec_and_test(katom)) { -+ kbase_fence_free_callbacks(katom); -+ kbase_fence_dep_count_set(katom, -1); -+ return 0; /* Already signaled, good to go right now */ -+ } -+ -+ /* Callback installed, so we just need to wait for it... */ -+ } else { -+ /* Failure */ -+ kbase_fence_free_callbacks(katom); -+ kbase_fence_dep_count_set(katom, -1); -+ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ /* We should cause the dependent jobs in the bag to be failed, -+ * to do this we schedule the work queue to complete this job */ -+ -+ INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); -+ queue_work(katom->kctx->jctx.job_done_wq, &katom->work); -+ } -+ -+ return 1; /* completion to be done later by callback/worker */ -+} -+ -+void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) -+{ -+ if (!kbase_fence_free_callbacks(katom)) { -+ /* The wait wasn't cancelled - -+ * leave the cleanup for kbase_fence_wait_callback */ -+ return; -+ } -+ -+ /* Take responsibility of completion */ -+ kbase_fence_dep_count_set(katom, -1); -+ -+ /* Wait was cancelled - zap the atoms */ -+ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; -+ -+ kbasep_remove_waiting_soft_job(katom); -+ kbase_finish_soft_job(katom); -+ -+ if (jd_done_nolock(katom, NULL)) -+ kbase_js_sched_all(katom->kctx->kbdev); -+} -+ -+void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) -+{ -+ kbase_fence_out_remove(katom); -+} -+ -+void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) -+{ -+ kbase_fence_free_callbacks(katom); -+ kbase_fence_in_remove(katom); -+} -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+static void kbase_sync_fence_info_get(struct fence *fence, -+ struct kbase_sync_fence_info *info) -+#else -+static void kbase_sync_fence_info_get(struct dma_fence *fence, -+ struct kbase_sync_fence_info *info) -+#endif -+{ -+ info->fence = fence; -+ -+ /* translate into CONFIG_SYNC status: -+ * < 0 : error -+ * 0 : active -+ * 1 : signaled -+ */ -+ if (dma_fence_is_signaled(fence)) { -+#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ -+ (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ -+ KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) -+ int status = fence->error; -+#else -+ int status = fence->status; -+#endif -+ if (status < 0) -+ info->status = status; /* signaled with error */ -+ else -+ info->status = 1; /* signaled with success */ -+ } else { -+ info->status = 0; /* still active (unsignaled) */ -+ } -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+ scnprintf(info->name, sizeof(info->name), "%u#%u", -+ fence->context, fence->seqno); -+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)) -+ scnprintf(info->name, sizeof(info->name), "%llu#%u", -+ fence->context, fence->seqno); -+#else -+ scnprintf(info->name, sizeof(info->name), "%llu#%llu", -+ fence->context, fence->seqno); -+#endif -+} -+ -+int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, -+ struct kbase_sync_fence_info *info) -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ -+ fence = kbase_fence_in_get(katom); -+ if (!fence) -+ return -ENOENT; -+ -+ kbase_sync_fence_info_get(fence, info); -+ -+ kbase_fence_put(fence); -+ -+ return 0; -+} -+ -+int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, -+ struct kbase_sync_fence_info *info) -+{ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ -+ fence = kbase_fence_out_get(katom); -+ if (!fence) -+ return -ENOENT; -+ -+ kbase_sync_fence_info_get(fence, info); -+ -+ kbase_fence_put(fence); -+ -+ return 0; -+} -+ -+ -+#ifdef CONFIG_MALI_FENCE_DEBUG -+void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) -+{ -+ /* Not implemented */ -+} -+#endif -diff --git a/drivers/gpu/arm/midgard/mali_kbase_tlstream.c b/drivers/gpu/arm/midgard/mali_kbase_tlstream.c -new file mode 100755 -index 000000000..c8310c45f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_tlstream.c -@@ -0,0 +1,2572 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+ -+/*****************************************************************************/ -+ -+/* The version of swtrace protocol used in timeline stream. */ -+#define SWTRACE_VERSION 3 -+ -+/* The maximum expected length of string in tracepoint descriptor. */ -+#define STRLEN_MAX 64 /* bytes */ -+ -+/* The number of nanoseconds in a second. */ -+#define NSECS_IN_SEC 1000000000ull /* ns */ -+ -+/* The period of autoflush checker execution in milliseconds. */ -+#define AUTOFLUSH_INTERVAL 1000 /* ms */ -+ -+/* The maximum size of a single packet used by timeline. */ -+#define PACKET_SIZE 4096 /* bytes */ -+ -+/* The number of packets used by one timeline stream. */ -+#define PACKET_COUNT 16 -+ -+/* The number of bytes reserved for packet header. -+ * These value must be defined according to MIPE documentation. */ -+#define PACKET_HEADER_SIZE 8 /* bytes */ -+ -+/* The number of bytes reserved for packet sequence number. -+ * These value must be defined according to MIPE documentation. */ -+#define PACKET_NUMBER_SIZE 4 /* bytes */ -+ -+/* Packet header - first word. -+ * These values must be defined according to MIPE documentation. */ -+#define PACKET_STREAMID_POS 0 -+#define PACKET_STREAMID_LEN 8 -+#define PACKET_RSVD1_POS (PACKET_STREAMID_POS + PACKET_STREAMID_LEN) -+#define PACKET_RSVD1_LEN 8 -+#define PACKET_TYPE_POS (PACKET_RSVD1_POS + PACKET_RSVD1_LEN) -+#define PACKET_TYPE_LEN 3 -+#define PACKET_CLASS_POS (PACKET_TYPE_POS + PACKET_TYPE_LEN) -+#define PACKET_CLASS_LEN 7 -+#define PACKET_FAMILY_POS (PACKET_CLASS_POS + PACKET_CLASS_LEN) -+#define PACKET_FAMILY_LEN 6 -+ -+/* Packet header - second word -+ * These values must be defined according to MIPE documentation. */ -+#define PACKET_LENGTH_POS 0 -+#define PACKET_LENGTH_LEN 24 -+#define PACKET_SEQBIT_POS (PACKET_LENGTH_POS + PACKET_LENGTH_LEN) -+#define PACKET_SEQBIT_LEN 1 -+#define PACKET_RSVD2_POS (PACKET_SEQBIT_POS + PACKET_SEQBIT_LEN) -+#define PACKET_RSVD2_LEN 7 -+ -+/* Types of streams generated by timeline. -+ * Order is significant! Header streams must precede respective body streams. */ -+enum tl_stream_type { -+ TL_STREAM_TYPE_OBJ_HEADER, -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ TL_STREAM_TYPE_OBJ, -+ TL_STREAM_TYPE_AUX_HEADER, -+ TL_STREAM_TYPE_AUX, -+ -+ TL_STREAM_TYPE_COUNT -+}; -+ -+/* Timeline packet family ids. -+ * Values are significant! Check MIPE documentation. */ -+enum tl_packet_family { -+ TL_PACKET_FAMILY_CTRL = 0, /* control packets */ -+ TL_PACKET_FAMILY_TL = 1, /* timeline packets */ -+ -+ TL_PACKET_FAMILY_COUNT -+}; -+ -+/* Packet classes used in timeline streams. -+ * Values are significant! Check MIPE documentation. */ -+enum tl_packet_class { -+ TL_PACKET_CLASS_OBJ = 0, /* timeline objects packet */ -+ TL_PACKET_CLASS_AUX = 1, /* auxiliary events packet */ -+}; -+ -+/* Packet types used in timeline streams. -+ * Values are significant! Check MIPE documentation. */ -+enum tl_packet_type { -+ TL_PACKET_TYPE_HEADER = 0, /* stream's header/directory */ -+ TL_PACKET_TYPE_BODY = 1, /* stream's body */ -+ TL_PACKET_TYPE_SUMMARY = 2, /* stream's summary */ -+}; -+ -+/* Message ids of trace events that are recorded in the timeline stream. */ -+enum tl_msg_id_obj { -+ /* Timeline object events. */ -+ KBASE_TL_NEW_CTX, -+ KBASE_TL_NEW_GPU, -+ KBASE_TL_NEW_LPU, -+ KBASE_TL_NEW_ATOM, -+ KBASE_TL_NEW_AS, -+ KBASE_TL_DEL_CTX, -+ KBASE_TL_DEL_ATOM, -+ KBASE_TL_LIFELINK_LPU_GPU, -+ KBASE_TL_LIFELINK_AS_GPU, -+ KBASE_TL_RET_CTX_LPU, -+ KBASE_TL_RET_ATOM_CTX, -+ KBASE_TL_RET_ATOM_LPU, -+ KBASE_TL_NRET_CTX_LPU, -+ KBASE_TL_NRET_ATOM_CTX, -+ KBASE_TL_NRET_ATOM_LPU, -+ KBASE_TL_RET_AS_CTX, -+ KBASE_TL_NRET_AS_CTX, -+ KBASE_TL_RET_ATOM_AS, -+ KBASE_TL_NRET_ATOM_AS, -+ KBASE_TL_DEP_ATOM_ATOM, -+ KBASE_TL_NDEP_ATOM_ATOM, -+ KBASE_TL_RDEP_ATOM_ATOM, -+ KBASE_TL_ATTRIB_ATOM_CONFIG, -+ KBASE_TL_ATTRIB_ATOM_PRIORITY, -+ KBASE_TL_ATTRIB_ATOM_STATE, -+ KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, -+ KBASE_TL_ATTRIB_ATOM_JIT, -+ KBASE_TL_ATTRIB_AS_CONFIG, -+ KBASE_TL_EVENT_LPU_SOFTSTOP, -+ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, -+ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, -+ -+ /* Job dump specific events. */ -+ KBASE_JD_GPU_SOFT_RESET -+}; -+ -+/* Message ids of trace events that are recorded in the auxiliary stream. */ -+enum tl_msg_id_aux { -+ KBASE_AUX_PM_STATE, -+ KBASE_AUX_PAGEFAULT, -+ KBASE_AUX_PAGESALLOC, -+ KBASE_AUX_DEVFREQ_TARGET, -+ KBASE_AUX_PROTECTED_ENTER_START, -+ KBASE_AUX_PROTECTED_ENTER_END, -+ KBASE_AUX_PROTECTED_LEAVE_START, -+ KBASE_AUX_PROTECTED_LEAVE_END -+}; -+ -+/*****************************************************************************/ -+ -+/** -+ * struct tl_stream - timeline stream structure -+ * @lock: message order lock -+ * @buffer: array of buffers -+ * @wbi: write buffer index -+ * @rbi: read buffer index -+ * @numbered: if non-zero stream's packets are sequentially numbered -+ * @autoflush_counter: counter tracking stream's autoflush state -+ * -+ * This structure holds information needed to construct proper packets in the -+ * timeline stream. Each message in sequence must bear timestamp that is greater -+ * to one in previous message in the same stream. For this reason lock is held -+ * throughout the process of message creation. Each stream contains set of -+ * buffers. Each buffer will hold one MIPE packet. In case there is no free -+ * space required to store incoming message the oldest buffer is discarded. -+ * Each packet in timeline body stream has sequence number embedded (this value -+ * must increment monotonically and is used by packets receiver to discover -+ * buffer overflows. -+ * Autoflush counter is set to negative number when there is no data pending -+ * for flush and it is set to zero on every update of the buffer. Autoflush -+ * timer will increment the counter by one on every expiry. In case there will -+ * be no activity on the buffer during two consecutive timer expiries, stream -+ * buffer will be flushed. -+ */ -+struct tl_stream { -+ spinlock_t lock; -+ -+ struct { -+ atomic_t size; /* number of bytes in buffer */ -+ char data[PACKET_SIZE]; /* buffer's data */ -+ } buffer[PACKET_COUNT]; -+ -+ atomic_t wbi; -+ atomic_t rbi; -+ -+ int numbered; -+ atomic_t autoflush_counter; -+}; -+ -+/** -+ * struct tp_desc - tracepoint message descriptor structure -+ * @id: tracepoint ID identifying message in stream -+ * @id_str: human readable version of tracepoint ID -+ * @name: tracepoint description -+ * @arg_types: tracepoint's arguments types declaration -+ * @arg_names: comma separated list of tracepoint's arguments names -+ */ -+struct tp_desc { -+ u32 id; -+ const char *id_str; -+ const char *name; -+ const char *arg_types; -+ const char *arg_names; -+}; -+ -+/*****************************************************************************/ -+ -+/* Configuration of timeline streams generated by kernel. -+ * Kernel emit only streams containing either timeline object events or -+ * auxiliary events. All streams have stream id value of 1 (as opposed to user -+ * space streams that have value of 0). */ -+static const struct { -+ enum tl_packet_family pkt_family; -+ enum tl_packet_class pkt_class; -+ enum tl_packet_type pkt_type; -+ unsigned int stream_id; -+} tl_stream_cfg[TL_STREAM_TYPE_COUNT] = { -+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_HEADER, 1}, -+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_SUMMARY, 1}, -+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, 1}, -+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_HEADER, 1}, -+ {TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_AUX, TL_PACKET_TYPE_BODY, 1} -+}; -+ -+/* The timeline streams generated by kernel. */ -+static struct tl_stream *tl_stream[TL_STREAM_TYPE_COUNT]; -+ -+/* Autoflush timer. */ -+static struct timer_list autoflush_timer; -+ -+/* If non-zero autoflush timer is active. */ -+static atomic_t autoflush_timer_active; -+ -+/* Reader lock. Only one reader is allowed to have access to the timeline -+ * streams at any given time. */ -+static DEFINE_MUTEX(tl_reader_lock); -+ -+/* Timeline stream event queue. */ -+static DECLARE_WAIT_QUEUE_HEAD(tl_event_queue); -+ -+/* The timeline stream file operations functions. */ -+static ssize_t kbasep_tlstream_read( -+ struct file *filp, -+ char __user *buffer, -+ size_t size, -+ loff_t *f_pos); -+static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait); -+static int kbasep_tlstream_release(struct inode *inode, struct file *filp); -+ -+/* The timeline stream file operations structure. */ -+static const struct file_operations kbasep_tlstream_fops = { -+ .release = kbasep_tlstream_release, -+ .read = kbasep_tlstream_read, -+ .poll = kbasep_tlstream_poll, -+}; -+ -+/* Descriptors of timeline messages transmitted in object events stream. */ -+static const struct tp_desc tp_desc_obj[] = { -+ { -+ KBASE_TL_NEW_CTX, -+ __stringify(KBASE_TL_NEW_CTX), -+ "object ctx is created", -+ "@pII", -+ "ctx,ctx_nr,tgid" -+ }, -+ { -+ KBASE_TL_NEW_GPU, -+ __stringify(KBASE_TL_NEW_GPU), -+ "object gpu is created", -+ "@pII", -+ "gpu,gpu_id,core_count" -+ }, -+ { -+ KBASE_TL_NEW_LPU, -+ __stringify(KBASE_TL_NEW_LPU), -+ "object lpu is created", -+ "@pII", -+ "lpu,lpu_nr,lpu_fn" -+ }, -+ { -+ KBASE_TL_NEW_ATOM, -+ __stringify(KBASE_TL_NEW_ATOM), -+ "object atom is created", -+ "@pI", -+ "atom,atom_nr" -+ }, -+ { -+ KBASE_TL_NEW_AS, -+ __stringify(KBASE_TL_NEW_AS), -+ "address space object is created", -+ "@pI", -+ "address_space,as_nr" -+ }, -+ { -+ KBASE_TL_DEL_CTX, -+ __stringify(KBASE_TL_DEL_CTX), -+ "context is destroyed", -+ "@p", -+ "ctx" -+ }, -+ { -+ KBASE_TL_DEL_ATOM, -+ __stringify(KBASE_TL_DEL_ATOM), -+ "atom is destroyed", -+ "@p", -+ "atom" -+ }, -+ { -+ KBASE_TL_LIFELINK_LPU_GPU, -+ __stringify(KBASE_TL_LIFELINK_LPU_GPU), -+ "lpu is deleted with gpu", -+ "@pp", -+ "lpu,gpu" -+ }, -+ { -+ KBASE_TL_LIFELINK_AS_GPU, -+ __stringify(KBASE_TL_LIFELINK_AS_GPU), -+ "address space is deleted with gpu", -+ "@pp", -+ "address_space,gpu" -+ }, -+ { -+ KBASE_TL_RET_CTX_LPU, -+ __stringify(KBASE_TL_RET_CTX_LPU), -+ "context is retained by lpu", -+ "@pp", -+ "ctx,lpu" -+ }, -+ { -+ KBASE_TL_RET_ATOM_CTX, -+ __stringify(KBASE_TL_RET_ATOM_CTX), -+ "atom is retained by context", -+ "@pp", -+ "atom,ctx" -+ }, -+ { -+ KBASE_TL_RET_ATOM_LPU, -+ __stringify(KBASE_TL_RET_ATOM_LPU), -+ "atom is retained by lpu", -+ "@pps", -+ "atom,lpu,attrib_match_list" -+ }, -+ { -+ KBASE_TL_NRET_CTX_LPU, -+ __stringify(KBASE_TL_NRET_CTX_LPU), -+ "context is released by lpu", -+ "@pp", -+ "ctx,lpu" -+ }, -+ { -+ KBASE_TL_NRET_ATOM_CTX, -+ __stringify(KBASE_TL_NRET_ATOM_CTX), -+ "atom is released by context", -+ "@pp", -+ "atom,ctx" -+ }, -+ { -+ KBASE_TL_NRET_ATOM_LPU, -+ __stringify(KBASE_TL_NRET_ATOM_LPU), -+ "atom is released by lpu", -+ "@pp", -+ "atom,lpu" -+ }, -+ { -+ KBASE_TL_RET_AS_CTX, -+ __stringify(KBASE_TL_RET_AS_CTX), -+ "address space is retained by context", -+ "@pp", -+ "address_space,ctx" -+ }, -+ { -+ KBASE_TL_NRET_AS_CTX, -+ __stringify(KBASE_TL_NRET_AS_CTX), -+ "address space is released by context", -+ "@pp", -+ "address_space,ctx" -+ }, -+ { -+ KBASE_TL_RET_ATOM_AS, -+ __stringify(KBASE_TL_RET_ATOM_AS), -+ "atom is retained by address space", -+ "@pp", -+ "atom,address_space" -+ }, -+ { -+ KBASE_TL_NRET_ATOM_AS, -+ __stringify(KBASE_TL_NRET_ATOM_AS), -+ "atom is released by address space", -+ "@pp", -+ "atom,address_space" -+ }, -+ { -+ KBASE_TL_DEP_ATOM_ATOM, -+ __stringify(KBASE_TL_DEP_ATOM_ATOM), -+ "atom2 depends on atom1", -+ "@pp", -+ "atom1,atom2" -+ }, -+ { -+ KBASE_TL_NDEP_ATOM_ATOM, -+ __stringify(KBASE_TL_NDEP_ATOM_ATOM), -+ "atom2 no longer depends on atom1", -+ "@pp", -+ "atom1,atom2" -+ }, -+ { -+ KBASE_TL_RDEP_ATOM_ATOM, -+ __stringify(KBASE_TL_RDEP_ATOM_ATOM), -+ "resolved dependecy of atom2 depending on atom1", -+ "@pp", -+ "atom1,atom2" -+ }, -+ { -+ KBASE_TL_ATTRIB_ATOM_CONFIG, -+ __stringify(KBASE_TL_ATTRIB_ATOM_CONFIG), -+ "atom job slot attributes", -+ "@pLLI", -+ "atom,descriptor,affinity,config" -+ }, -+ { -+ KBASE_TL_ATTRIB_ATOM_PRIORITY, -+ __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY), -+ "atom priority", -+ "@pI", -+ "atom,prio" -+ }, -+ { -+ KBASE_TL_ATTRIB_ATOM_STATE, -+ __stringify(KBASE_TL_ATTRIB_ATOM_STATE), -+ "atom state", -+ "@pI", -+ "atom,state" -+ }, -+ { -+ KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE, -+ __stringify(KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE), -+ "atom caused priority change", -+ "@p", -+ "atom" -+ }, -+ { -+ KBASE_TL_ATTRIB_ATOM_JIT, -+ __stringify(KBASE_TL_ATTRIB_ATOM_JIT), -+ "jit done for atom", -+ "@pLL", -+ "atom,edit_addr,new_addr" -+ }, -+ { -+ KBASE_TL_ATTRIB_AS_CONFIG, -+ __stringify(KBASE_TL_ATTRIB_AS_CONFIG), -+ "address space attributes", -+ "@pLLL", -+ "address_space,transtab,memattr,transcfg" -+ }, -+ { -+ KBASE_TL_EVENT_LPU_SOFTSTOP, -+ __stringify(KBASE_TL_EVENT_LPU_SOFTSTOP), -+ "softstop event on given lpu", -+ "@p", -+ "lpu" -+ }, -+ { -+ KBASE_TL_EVENT_ATOM_SOFTSTOP_EX, -+ __stringify(KBASE_TL_EVENT_ATOM_SOFTSTOP_EX), -+ "atom softstopped", -+ "@p", -+ "atom" -+ }, -+ { -+ KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, -+ __stringify(KBASE_TL_EVENT_SOFTSTOP_ISSUE), -+ "atom softstop issued", -+ "@p", -+ "atom" -+ }, -+ { -+ KBASE_JD_GPU_SOFT_RESET, -+ __stringify(KBASE_JD_GPU_SOFT_RESET), -+ "gpu soft reset", -+ "@p", -+ "gpu" -+ }, -+}; -+ -+/* Descriptors of timeline messages transmitted in auxiliary events stream. */ -+static const struct tp_desc tp_desc_aux[] = { -+ { -+ KBASE_AUX_PM_STATE, -+ __stringify(KBASE_AUX_PM_STATE), -+ "PM state", -+ "@IL", -+ "core_type,core_state_bitset" -+ }, -+ { -+ KBASE_AUX_PAGEFAULT, -+ __stringify(KBASE_AUX_PAGEFAULT), -+ "Page fault", -+ "@IL", -+ "ctx_nr,page_cnt_change" -+ }, -+ { -+ KBASE_AUX_PAGESALLOC, -+ __stringify(KBASE_AUX_PAGESALLOC), -+ "Total alloc pages change", -+ "@IL", -+ "ctx_nr,page_cnt" -+ }, -+ { -+ KBASE_AUX_DEVFREQ_TARGET, -+ __stringify(KBASE_AUX_DEVFREQ_TARGET), -+ "New device frequency target", -+ "@L", -+ "target_freq" -+ }, -+ { -+ KBASE_AUX_PROTECTED_ENTER_START, -+ __stringify(KBASE_AUX_PROTECTED_ENTER_START), -+ "enter protected mode start", -+ "@p", -+ "gpu" -+ }, -+ { -+ KBASE_AUX_PROTECTED_ENTER_END, -+ __stringify(KBASE_AUX_PROTECTED_ENTER_END), -+ "enter protected mode end", -+ "@p", -+ "gpu" -+ }, -+ { -+ KBASE_AUX_PROTECTED_LEAVE_START, -+ __stringify(KBASE_AUX_PROTECTED_LEAVE_START), -+ "leave protected mode start", -+ "@p", -+ "gpu" -+ }, -+ { -+ KBASE_AUX_PROTECTED_LEAVE_END, -+ __stringify(KBASE_AUX_PROTECTED_LEAVE_END), -+ "leave protected mode end", -+ "@p", -+ "gpu" -+ } -+}; -+ -+#if MALI_UNIT_TEST -+/* Number of bytes read by user. */ -+static atomic_t tlstream_bytes_collected = {0}; -+ -+/* Number of bytes generated by tracepoint messages. */ -+static atomic_t tlstream_bytes_generated = {0}; -+#endif /* MALI_UNIT_TEST */ -+ -+/*****************************************************************************/ -+ -+/* Indicator of whether the timeline stream file descriptor is used. */ -+atomic_t kbase_tlstream_enabled = {0}; -+ -+/*****************************************************************************/ -+ -+/** -+ * kbasep_tlstream_get_timestamp - return timestamp -+ * -+ * Function returns timestamp value based on raw monotonic timer. Value will -+ * wrap around zero in case of overflow. -+ * Return: timestamp value -+ */ -+static u64 kbasep_tlstream_get_timestamp(void) -+{ -+ struct timespec64 ts; -+ u64 timestamp; -+ -+ ktime_get_raw_ts64(&ts); -+ timestamp = (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; -+ return timestamp; -+} -+ -+/** -+ * kbasep_tlstream_write_bytes - write data to message buffer -+ * @buffer: buffer where data will be written -+ * @pos: position in the buffer where to place data -+ * @bytes: pointer to buffer holding data -+ * @len: length of data to be written -+ * -+ * Return: updated position in the buffer -+ */ -+static size_t kbasep_tlstream_write_bytes( -+ char *buffer, -+ size_t pos, -+ const void *bytes, -+ size_t len) -+{ -+ KBASE_DEBUG_ASSERT(buffer); -+ KBASE_DEBUG_ASSERT(bytes); -+ -+ memcpy(&buffer[pos], bytes, len); -+ -+ return pos + len; -+} -+ -+/** -+ * kbasep_tlstream_write_string - write string to message buffer -+ * @buffer: buffer where data will be written -+ * @pos: position in the buffer where to place data -+ * @string: pointer to buffer holding the source string -+ * @max_write_size: number of bytes that can be stored in buffer -+ * -+ * Return: updated position in the buffer -+ */ -+static size_t kbasep_tlstream_write_string( -+ char *buffer, -+ size_t pos, -+ const char *string, -+ size_t max_write_size) -+{ -+ u32 string_len; -+ -+ KBASE_DEBUG_ASSERT(buffer); -+ KBASE_DEBUG_ASSERT(string); -+ /* Timeline string consists of at least string length and nul -+ * terminator. */ -+ KBASE_DEBUG_ASSERT(max_write_size >= sizeof(string_len) + sizeof(char)); -+ max_write_size -= sizeof(string_len); -+ -+ string_len = strlcpy( -+ &buffer[pos + sizeof(string_len)], -+ string, -+ max_write_size); -+ string_len += sizeof(char); -+ -+ /* Make sure that the source string fit into the buffer. */ -+ KBASE_DEBUG_ASSERT(string_len <= max_write_size); -+ -+ /* Update string length. */ -+ memcpy(&buffer[pos], &string_len, sizeof(string_len)); -+ -+ return pos + sizeof(string_len) + string_len; -+} -+ -+/** -+ * kbasep_tlstream_write_timestamp - write timestamp to message buffer -+ * @buffer: buffer where data will be written -+ * @pos: position in the buffer where to place data -+ * -+ * Return: updated position in the buffer -+ */ -+static size_t kbasep_tlstream_write_timestamp(void *buffer, size_t pos) -+{ -+ u64 timestamp = kbasep_tlstream_get_timestamp(); -+ -+ return kbasep_tlstream_write_bytes( -+ buffer, pos, -+ ×tamp, sizeof(timestamp)); -+} -+ -+/** -+ * kbasep_tlstream_put_bits - put bits in a word -+ * @word: pointer to the words being modified -+ * @value: value that shall be written to given position -+ * @bitpos: position where value shall be written (in bits) -+ * @bitlen: length of value (in bits) -+ */ -+static void kbasep_tlstream_put_bits( -+ u32 *word, -+ u32 value, -+ unsigned int bitpos, -+ unsigned int bitlen) -+{ -+ const u32 mask = ((1 << bitlen) - 1) << bitpos; -+ -+ KBASE_DEBUG_ASSERT(word); -+ KBASE_DEBUG_ASSERT((0 != bitlen) && (32 >= bitlen)); -+ KBASE_DEBUG_ASSERT((bitpos + bitlen) <= 32); -+ -+ *word &= ~mask; -+ *word |= ((value << bitpos) & mask); -+} -+ -+/** -+ * kbasep_tlstream_packet_header_setup - setup the packet header -+ * @buffer: pointer to the buffer -+ * @pkt_family: packet's family -+ * @pkt_type: packet's type -+ * @pkt_class: packet's class -+ * @stream_id: stream id -+ * @numbered: non-zero if this stream is numbered -+ * -+ * Function sets up immutable part of packet header in the given buffer. -+ */ -+static void kbasep_tlstream_packet_header_setup( -+ char *buffer, -+ enum tl_packet_family pkt_family, -+ enum tl_packet_class pkt_class, -+ enum tl_packet_type pkt_type, -+ unsigned int stream_id, -+ int numbered) -+{ -+ u32 word0 = 0; -+ u32 word1 = 0; -+ -+ KBASE_DEBUG_ASSERT(buffer); -+ KBASE_DEBUG_ASSERT(pkt_family == TL_PACKET_FAMILY_TL); -+ KBASE_DEBUG_ASSERT( -+ (pkt_type == TL_PACKET_TYPE_HEADER) || -+ (pkt_type == TL_PACKET_TYPE_SUMMARY) || -+ (pkt_type == TL_PACKET_TYPE_BODY)); -+ KBASE_DEBUG_ASSERT( -+ (pkt_class == TL_PACKET_CLASS_OBJ) || -+ (pkt_class == TL_PACKET_CLASS_AUX)); -+ -+ kbasep_tlstream_put_bits( -+ &word0, pkt_family, -+ PACKET_FAMILY_POS, PACKET_FAMILY_LEN); -+ kbasep_tlstream_put_bits( -+ &word0, pkt_class, -+ PACKET_CLASS_POS, PACKET_CLASS_LEN); -+ kbasep_tlstream_put_bits( -+ &word0, pkt_type, -+ PACKET_TYPE_POS, PACKET_TYPE_LEN); -+ kbasep_tlstream_put_bits( -+ &word0, stream_id, -+ PACKET_STREAMID_POS, PACKET_STREAMID_LEN); -+ -+ if (numbered) -+ kbasep_tlstream_put_bits( -+ &word1, 1, -+ PACKET_SEQBIT_POS, PACKET_SEQBIT_LEN); -+ -+ memcpy(&buffer[0], &word0, sizeof(word0)); -+ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); -+} -+ -+/** -+ * kbasep_tlstream_packet_header_update - update the packet header -+ * @buffer: pointer to the buffer -+ * @data_size: amount of data carried in this packet -+ * -+ * Function updates mutable part of packet header in the given buffer. -+ * Note that value of data_size must not including size of the header. -+ */ -+static void kbasep_tlstream_packet_header_update( -+ char *buffer, -+ size_t data_size) -+{ -+ u32 word0; -+ u32 word1; -+ -+ KBASE_DEBUG_ASSERT(buffer); -+ CSTD_UNUSED(word0); -+ -+ memcpy(&word1, &buffer[sizeof(word0)], sizeof(word1)); -+ -+ kbasep_tlstream_put_bits( -+ &word1, data_size, -+ PACKET_LENGTH_POS, PACKET_LENGTH_LEN); -+ -+ memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); -+} -+ -+/** -+ * kbasep_tlstream_packet_number_update - update the packet number -+ * @buffer: pointer to the buffer -+ * @counter: value of packet counter for this packet's stream -+ * -+ * Function updates packet number embedded within the packet placed in the -+ * given buffer. -+ */ -+static void kbasep_tlstream_packet_number_update(char *buffer, u32 counter) -+{ -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ memcpy(&buffer[PACKET_HEADER_SIZE], &counter, sizeof(counter)); -+} -+ -+/** -+ * kbasep_timeline_stream_reset - reset stream -+ * @stream: pointer to the stream structure -+ * -+ * Function discards all pending messages and resets packet counters. -+ */ -+static void kbasep_timeline_stream_reset(struct tl_stream *stream) -+{ -+ unsigned int i; -+ -+ for (i = 0; i < PACKET_COUNT; i++) { -+ if (stream->numbered) -+ atomic_set( -+ &stream->buffer[i].size, -+ PACKET_HEADER_SIZE + -+ PACKET_NUMBER_SIZE); -+ else -+ atomic_set(&stream->buffer[i].size, PACKET_HEADER_SIZE); -+ } -+ -+ atomic_set(&stream->wbi, 0); -+ atomic_set(&stream->rbi, 0); -+} -+ -+/** -+ * kbasep_timeline_stream_init - initialize timeline stream -+ * @stream: pointer to the stream structure -+ * @stream_type: stream type -+ */ -+static void kbasep_timeline_stream_init( -+ struct tl_stream *stream, -+ enum tl_stream_type stream_type) -+{ -+ unsigned int i; -+ -+ KBASE_DEBUG_ASSERT(stream); -+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); -+ -+ spin_lock_init(&stream->lock); -+ -+ /* All packets carrying tracepoints shall be numbered. */ -+ if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) -+ stream->numbered = 1; -+ else -+ stream->numbered = 0; -+ -+ for (i = 0; i < PACKET_COUNT; i++) -+ kbasep_tlstream_packet_header_setup( -+ stream->buffer[i].data, -+ tl_stream_cfg[stream_type].pkt_family, -+ tl_stream_cfg[stream_type].pkt_class, -+ tl_stream_cfg[stream_type].pkt_type, -+ tl_stream_cfg[stream_type].stream_id, -+ stream->numbered); -+ -+ kbasep_timeline_stream_reset(tl_stream[stream_type]); -+} -+ -+/** -+ * kbasep_timeline_stream_term - terminate timeline stream -+ * @stream: pointer to the stream structure -+ */ -+static void kbasep_timeline_stream_term(struct tl_stream *stream) -+{ -+ KBASE_DEBUG_ASSERT(stream); -+} -+ -+/** -+ * kbasep_tlstream_msgbuf_submit - submit packet to the user space -+ * @stream: pointer to the stream structure -+ * @wb_idx_raw: write buffer index -+ * @wb_size: length of data stored in current buffer -+ * -+ * Function updates currently written buffer with packet header. Then write -+ * index is incremented and buffer is handled to user space. Parameters -+ * of new buffer are returned using provided arguments. -+ * -+ * Return: length of data in new buffer -+ * -+ * Warning: User must update the stream structure with returned value. -+ */ -+static size_t kbasep_tlstream_msgbuf_submit( -+ struct tl_stream *stream, -+ unsigned int wb_idx_raw, -+ unsigned int wb_size) -+{ -+ unsigned int rb_idx_raw = atomic_read(&stream->rbi); -+ unsigned int wb_idx = wb_idx_raw % PACKET_COUNT; -+ -+ /* Set stream as flushed. */ -+ atomic_set(&stream->autoflush_counter, -1); -+ -+ kbasep_tlstream_packet_header_update( -+ stream->buffer[wb_idx].data, -+ wb_size - PACKET_HEADER_SIZE); -+ -+ if (stream->numbered) -+ kbasep_tlstream_packet_number_update( -+ stream->buffer[wb_idx].data, -+ wb_idx_raw); -+ -+ /* Increasing write buffer index will expose this packet to the reader. -+ * As stream->lock is not taken on reader side we must make sure memory -+ * is updated correctly before this will happen. */ -+ smp_wmb(); -+ wb_idx_raw++; -+ atomic_set(&stream->wbi, wb_idx_raw); -+ -+ /* Inform user that packets are ready for reading. */ -+ wake_up_interruptible(&tl_event_queue); -+ -+ /* Detect and mark overflow in this stream. */ -+ if (PACKET_COUNT == wb_idx_raw - rb_idx_raw) { -+ /* Reader side depends on this increment to correctly handle -+ * overflows. The value shall be updated only if it was not -+ * modified by the reader. The data holding buffer will not be -+ * updated before stream->lock is released, however size of the -+ * buffer will. Make sure this increment is globally visible -+ * before information about selected write buffer size. */ -+ atomic_cmpxchg(&stream->rbi, rb_idx_raw, rb_idx_raw + 1); -+ } -+ -+ wb_size = PACKET_HEADER_SIZE; -+ if (stream->numbered) -+ wb_size += PACKET_NUMBER_SIZE; -+ -+ return wb_size; -+} -+ -+/** -+ * kbasep_tlstream_msgbuf_acquire - lock selected stream and reserves buffer -+ * @stream_type: type of the stream that shall be locked -+ * @msg_size: message size -+ * @flags: pointer to store flags passed back on stream release -+ * -+ * Function will lock the stream and reserve the number of bytes requested -+ * in msg_size for the user. -+ * -+ * Return: pointer to the buffer where message can be stored -+ * -+ * Warning: Stream must be released with kbasep_tlstream_msgbuf_release(). -+ * Only atomic operations are allowed while stream is locked -+ * (i.e. do not use any operation that may sleep). -+ */ -+static char *kbasep_tlstream_msgbuf_acquire( -+ enum tl_stream_type stream_type, -+ size_t msg_size, -+ unsigned long *flags) __acquires(&stream->lock) -+{ -+ struct tl_stream *stream; -+ unsigned int wb_idx_raw; -+ unsigned int wb_idx; -+ size_t wb_size; -+ -+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); -+ KBASE_DEBUG_ASSERT( -+ PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= -+ msg_size); -+ -+ stream = tl_stream[stream_type]; -+ -+ spin_lock_irqsave(&stream->lock, *flags); -+ -+ wb_idx_raw = atomic_read(&stream->wbi); -+ wb_idx = wb_idx_raw % PACKET_COUNT; -+ wb_size = atomic_read(&stream->buffer[wb_idx].size); -+ -+ /* Select next buffer if data will not fit into current one. */ -+ if (PACKET_SIZE < wb_size + msg_size) { -+ wb_size = kbasep_tlstream_msgbuf_submit( -+ stream, wb_idx_raw, wb_size); -+ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; -+ } -+ -+ /* Reserve space in selected buffer. */ -+ atomic_set(&stream->buffer[wb_idx].size, wb_size + msg_size); -+ -+#if MALI_UNIT_TEST -+ atomic_add(msg_size, &tlstream_bytes_generated); -+#endif /* MALI_UNIT_TEST */ -+ -+ return &stream->buffer[wb_idx].data[wb_size]; -+} -+ -+/** -+ * kbasep_tlstream_msgbuf_release - unlock selected stream -+ * @stream_type: type of the stream that shall be locked -+ * @flags: value obtained during stream acquire -+ * -+ * Function releases stream that has been previously locked with a call to -+ * kbasep_tlstream_msgbuf_acquire(). -+ */ -+static void kbasep_tlstream_msgbuf_release( -+ enum tl_stream_type stream_type, -+ unsigned long flags) __releases(&stream->lock) -+{ -+ struct tl_stream *stream; -+ -+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); -+ -+ stream = tl_stream[stream_type]; -+ -+ /* Mark stream as containing unflushed data. */ -+ atomic_set(&stream->autoflush_counter, 0); -+ -+ spin_unlock_irqrestore(&stream->lock, flags); -+} -+ -+/*****************************************************************************/ -+ -+/** -+ * kbasep_tlstream_flush_stream - flush stream -+ * @stype: type of stream to be flushed -+ * -+ * Flush pending data in timeline stream. -+ */ -+static void kbasep_tlstream_flush_stream(enum tl_stream_type stype) -+{ -+ struct tl_stream *stream = tl_stream[stype]; -+ unsigned long flags; -+ unsigned int wb_idx_raw; -+ unsigned int wb_idx; -+ size_t wb_size; -+ size_t min_size = PACKET_HEADER_SIZE; -+ -+ if (stream->numbered) -+ min_size += PACKET_NUMBER_SIZE; -+ -+ spin_lock_irqsave(&stream->lock, flags); -+ -+ wb_idx_raw = atomic_read(&stream->wbi); -+ wb_idx = wb_idx_raw % PACKET_COUNT; -+ wb_size = atomic_read(&stream->buffer[wb_idx].size); -+ -+ if (wb_size > min_size) { -+ wb_size = kbasep_tlstream_msgbuf_submit( -+ stream, wb_idx_raw, wb_size); -+ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; -+ atomic_set(&stream->buffer[wb_idx].size, wb_size); -+ } -+ spin_unlock_irqrestore(&stream->lock, flags); -+} -+ -+/** -+ * kbasep_tlstream_autoflush_timer_callback - autoflush timer callback -+ * @data: unused -+ * -+ * Timer is executed periodically to check if any of the stream contains -+ * buffer ready to be submitted to user space. -+ */ -+static void kbasep_tlstream_autoflush_timer_callback(struct timer_list *t) -+{ -+ enum tl_stream_type stype; -+ int rcode; -+ -+ CSTD_UNUSED(t); -+ -+ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { -+ struct tl_stream *stream = tl_stream[stype]; -+ unsigned long flags; -+ unsigned int wb_idx_raw; -+ unsigned int wb_idx; -+ size_t wb_size; -+ size_t min_size = PACKET_HEADER_SIZE; -+ -+ int af_cnt = atomic_read(&stream->autoflush_counter); -+ -+ /* Check if stream contain unflushed data. */ -+ if (0 > af_cnt) -+ continue; -+ -+ /* Check if stream should be flushed now. */ -+ if (af_cnt != atomic_cmpxchg( -+ &stream->autoflush_counter, -+ af_cnt, -+ af_cnt + 1)) -+ continue; -+ if (!af_cnt) -+ continue; -+ -+ /* Autoflush this stream. */ -+ if (stream->numbered) -+ min_size += PACKET_NUMBER_SIZE; -+ -+ spin_lock_irqsave(&stream->lock, flags); -+ -+ wb_idx_raw = atomic_read(&stream->wbi); -+ wb_idx = wb_idx_raw % PACKET_COUNT; -+ wb_size = atomic_read(&stream->buffer[wb_idx].size); -+ -+ if (wb_size > min_size) { -+ wb_size = kbasep_tlstream_msgbuf_submit( -+ stream, wb_idx_raw, wb_size); -+ wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; -+ atomic_set(&stream->buffer[wb_idx].size, -+ wb_size); -+ } -+ spin_unlock_irqrestore(&stream->lock, flags); -+ } -+ -+ if (atomic_read(&autoflush_timer_active)) -+ rcode = mod_timer( -+ &autoflush_timer, -+ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); -+ CSTD_UNUSED(rcode); -+} -+ -+/** -+ * kbasep_tlstream_packet_pending - check timeline streams for pending packets -+ * @stype: pointer to variable where stream type will be placed -+ * @rb_idx_raw: pointer to variable where read buffer index will be placed -+ * -+ * Function checks all streams for pending packets. It will stop as soon as -+ * packet ready to be submitted to user space is detected. Variables under -+ * pointers, passed as the parameters to this function will be updated with -+ * values pointing to right stream and buffer. -+ * -+ * Return: non-zero if any of timeline streams has at last one packet ready -+ */ -+static int kbasep_tlstream_packet_pending( -+ enum tl_stream_type *stype, -+ unsigned int *rb_idx_raw) -+{ -+ int pending = 0; -+ -+ KBASE_DEBUG_ASSERT(stype); -+ KBASE_DEBUG_ASSERT(rb_idx_raw); -+ -+ for ( -+ *stype = 0; -+ (*stype < TL_STREAM_TYPE_COUNT) && !pending; -+ (*stype)++) { -+ if (NULL != tl_stream[*stype]) { -+ *rb_idx_raw = atomic_read(&tl_stream[*stype]->rbi); -+ /* Read buffer index may be updated by writer in case of -+ * overflow. Read and write buffer indexes must be -+ * loaded in correct order. */ -+ smp_rmb(); -+ if (atomic_read(&tl_stream[*stype]->wbi) != *rb_idx_raw) -+ pending = 1; -+ } -+ } -+ (*stype)--; -+ -+ return pending; -+} -+ -+/** -+ * kbasep_tlstream_read - copy data from streams to buffer provided by user -+ * @filp: pointer to file structure (unused) -+ * @buffer: pointer to the buffer provided by user -+ * @size: maximum amount of data that can be stored in the buffer -+ * @f_pos: pointer to file offset (unused) -+ * -+ * Return: number of bytes stored in the buffer -+ */ -+static ssize_t kbasep_tlstream_read( -+ struct file *filp, -+ char __user *buffer, -+ size_t size, -+ loff_t *f_pos) -+{ -+ ssize_t copy_len = 0; -+ -+ KBASE_DEBUG_ASSERT(filp); -+ KBASE_DEBUG_ASSERT(f_pos); -+ -+ if (!buffer) -+ return -EINVAL; -+ -+ if ((0 > *f_pos) || (PACKET_SIZE > size)) -+ return -EINVAL; -+ -+ mutex_lock(&tl_reader_lock); -+ -+ while (copy_len < size) { -+ enum tl_stream_type stype; -+ unsigned int rb_idx_raw = 0; -+ unsigned int rb_idx; -+ size_t rb_size; -+ -+ /* If we don't have any data yet, wait for packet to be -+ * submitted. If we already read some packets and there is no -+ * packet pending return back to user. */ -+ if (0 < copy_len) { -+ if (!kbasep_tlstream_packet_pending( -+ &stype, -+ &rb_idx_raw)) -+ break; -+ } else { -+ if (wait_event_interruptible( -+ tl_event_queue, -+ kbasep_tlstream_packet_pending( -+ &stype, -+ &rb_idx_raw))) { -+ copy_len = -ERESTARTSYS; -+ break; -+ } -+ } -+ -+ /* Check if this packet fits into the user buffer. -+ * If so copy its content. */ -+ rb_idx = rb_idx_raw % PACKET_COUNT; -+ rb_size = atomic_read(&tl_stream[stype]->buffer[rb_idx].size); -+ if (rb_size > size - copy_len) -+ break; -+ if (copy_to_user( -+ &buffer[copy_len], -+ tl_stream[stype]->buffer[rb_idx].data, -+ rb_size)) { -+ copy_len = -EFAULT; -+ break; -+ } -+ -+ /* If the rbi still points to the packet we just processed -+ * then there was no overflow so we add the copied size to -+ * copy_len and move rbi on to the next packet -+ */ -+ smp_rmb(); -+ if (atomic_read(&tl_stream[stype]->rbi) == rb_idx_raw) { -+ copy_len += rb_size; -+ atomic_inc(&tl_stream[stype]->rbi); -+ -+#if MALI_UNIT_TEST -+ atomic_add(rb_size, &tlstream_bytes_collected); -+#endif /* MALI_UNIT_TEST */ -+ } -+ } -+ -+ mutex_unlock(&tl_reader_lock); -+ -+ return copy_len; -+} -+ -+/** -+ * kbasep_tlstream_poll - poll timeline stream for packets -+ * @filp: pointer to file structure -+ * @wait: pointer to poll table -+ * Return: POLLIN if data can be read without blocking, otherwise zero -+ */ -+static unsigned int kbasep_tlstream_poll(struct file *filp, poll_table *wait) -+{ -+ enum tl_stream_type stream_type; -+ unsigned int rb_idx; -+ -+ KBASE_DEBUG_ASSERT(filp); -+ KBASE_DEBUG_ASSERT(wait); -+ -+ poll_wait(filp, &tl_event_queue, wait); -+ if (kbasep_tlstream_packet_pending(&stream_type, &rb_idx)) -+ return POLLIN; -+ return 0; -+} -+ -+/** -+ * kbasep_tlstream_release - release timeline stream descriptor -+ * @inode: pointer to inode structure -+ * @filp: pointer to file structure -+ * -+ * Return always return zero -+ */ -+static int kbasep_tlstream_release(struct inode *inode, struct file *filp) -+{ -+ KBASE_DEBUG_ASSERT(inode); -+ KBASE_DEBUG_ASSERT(filp); -+ CSTD_UNUSED(inode); -+ CSTD_UNUSED(filp); -+ -+ /* Stop autoflush timer before releasing access to streams. */ -+ atomic_set(&autoflush_timer_active, 0); -+ del_timer_sync(&autoflush_timer); -+ -+ atomic_set(&kbase_tlstream_enabled, 0); -+ return 0; -+} -+ -+/** -+ * kbasep_tlstream_timeline_header - prepare timeline header stream packet -+ * @stream_type: type of the stream that will carry header data -+ * @tp_desc: pointer to array with tracepoint descriptors -+ * @tp_count: number of descriptors in the given array -+ * -+ * Functions fills in information about tracepoints stored in body stream -+ * associated with this header stream. -+ */ -+static void kbasep_tlstream_timeline_header( -+ enum tl_stream_type stream_type, -+ const struct tp_desc *tp_desc, -+ u32 tp_count) -+{ -+ const u8 tv = SWTRACE_VERSION; /* protocol version */ -+ const u8 ps = sizeof(void *); /* pointer size */ -+ size_t msg_size = sizeof(tv) + sizeof(ps) + sizeof(tp_count); -+ char *buffer; -+ size_t pos = 0; -+ unsigned long flags; -+ unsigned int i; -+ -+ KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); -+ KBASE_DEBUG_ASSERT(tp_desc); -+ -+ /* Calculate the size of the timeline message. */ -+ for (i = 0; i < tp_count; i++) { -+ msg_size += sizeof(tp_desc[i].id); -+ msg_size += -+ strnlen(tp_desc[i].id_str, STRLEN_MAX) + -+ sizeof(char) + sizeof(u32); -+ msg_size += -+ strnlen(tp_desc[i].name, STRLEN_MAX) + -+ sizeof(char) + sizeof(u32); -+ msg_size += -+ strnlen(tp_desc[i].arg_types, STRLEN_MAX) + -+ sizeof(char) + sizeof(u32); -+ msg_size += -+ strnlen(tp_desc[i].arg_names, STRLEN_MAX) + -+ sizeof(char) + sizeof(u32); -+ } -+ -+ KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE >= msg_size); -+ -+ buffer = kbasep_tlstream_msgbuf_acquire(stream_type, msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &tv, sizeof(tv)); -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &ps, sizeof(ps)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &tp_count, sizeof(tp_count)); -+ -+ for (i = 0; i < tp_count; i++) { -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, -+ &tp_desc[i].id, sizeof(tp_desc[i].id)); -+ pos = kbasep_tlstream_write_string( -+ buffer, pos, -+ tp_desc[i].id_str, msg_size - pos); -+ pos = kbasep_tlstream_write_string( -+ buffer, pos, -+ tp_desc[i].name, msg_size - pos); -+ pos = kbasep_tlstream_write_string( -+ buffer, pos, -+ tp_desc[i].arg_types, msg_size - pos); -+ pos = kbasep_tlstream_write_string( -+ buffer, pos, -+ tp_desc[i].arg_names, msg_size - pos); -+ } -+ -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(stream_type, flags); -+ -+ /* We don't expect any more data to be read in this stream. -+ * As header stream must be read before its associated body stream, -+ * make this packet visible to the user straightaway. */ -+ kbasep_tlstream_flush_stream(stream_type); -+} -+ -+/*****************************************************************************/ -+ -+int kbase_tlstream_init(void) -+{ -+ enum tl_stream_type i; -+ -+ /* Prepare stream structures. */ -+ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { -+ tl_stream[i] = kmalloc(sizeof(**tl_stream), GFP_KERNEL); -+ if (!tl_stream[i]) -+ break; -+ kbasep_timeline_stream_init(tl_stream[i], i); -+ } -+ if (TL_STREAM_TYPE_COUNT > i) { -+ for (; i > 0; i--) { -+ kbasep_timeline_stream_term(tl_stream[i - 1]); -+ kfree(tl_stream[i - 1]); -+ } -+ return -ENOMEM; -+ } -+ -+ /* Initialize autoflush timer. */ -+ atomic_set(&autoflush_timer_active, 0); -+ timer_setup(&autoflush_timer, -+ kbasep_tlstream_autoflush_timer_callback, -+ 0); -+ -+ return 0; -+} -+ -+void kbase_tlstream_term(void) -+{ -+ enum tl_stream_type i; -+ -+ for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) { -+ kbasep_timeline_stream_term(tl_stream[i]); -+ kfree(tl_stream[i]); -+ } -+} -+ -+static void kbase_create_timeline_objects(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ unsigned int lpu_id; -+ unsigned int as_nr; -+ struct kbasep_kctx_list_element *element; -+ -+ /* Create LPU objects. */ -+ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { -+ u32 *lpu = -+ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; -+ KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, lpu_id, *lpu); -+ } -+ -+ /* Create Address Space objects. */ -+ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) -+ KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(&kbdev->as[as_nr], as_nr); -+ -+ /* Create GPU object and make it retain all LPUs and address spaces. */ -+ KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU( -+ kbdev, -+ kbdev->gpu_props.props.raw_props.gpu_id, -+ kbdev->gpu_props.num_cores); -+ -+ for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { -+ void *lpu = -+ &kbdev->gpu_props.props.raw_props.js_features[lpu_id]; -+ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, kbdev); -+ } -+ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) -+ KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU( -+ &kbdev->as[as_nr], -+ kbdev); -+ -+ /* Create object for each known context. */ -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_for_each_entry(element, &kbdev->kctx_list, link) { -+ KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX( -+ element->kctx, -+ (u32)(element->kctx->id), -+ (u32)(element->kctx->tgid)); -+ } -+ /* Before releasing the lock, reset body stream buffers. -+ * This will prevent context creation message to be directed to both -+ * summary and body stream. -+ */ -+ kbase_tlstream_reset_body_streams(); -+ mutex_unlock(&kbdev->kctx_list_lock); -+ /* Static object are placed into summary packet that needs to be -+ * transmitted first. Flush all streams to make it available to -+ * user space. -+ */ -+ kbase_tlstream_flush_streams(); -+} -+ -+int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags) -+{ -+ int ret; -+ u32 tlstream_enabled = TLSTREAM_ENABLED | flags; -+ -+ if (0 == atomic_cmpxchg(&kbase_tlstream_enabled, 0, tlstream_enabled)) { -+ int rcode; -+ -+ ret = anon_inode_getfd( -+ "[mali_tlstream]", -+ &kbasep_tlstream_fops, -+ kctx, -+ O_RDONLY | O_CLOEXEC); -+ if (ret < 0) { -+ atomic_set(&kbase_tlstream_enabled, 0); -+ return ret; -+ } -+ -+ /* Reset and initialize header streams. */ -+ kbasep_timeline_stream_reset( -+ tl_stream[TL_STREAM_TYPE_OBJ_HEADER]); -+ kbasep_timeline_stream_reset( -+ tl_stream[TL_STREAM_TYPE_OBJ_SUMMARY]); -+ kbasep_timeline_stream_reset( -+ tl_stream[TL_STREAM_TYPE_AUX_HEADER]); -+ kbasep_tlstream_timeline_header( -+ TL_STREAM_TYPE_OBJ_HEADER, -+ tp_desc_obj, -+ ARRAY_SIZE(tp_desc_obj)); -+ kbasep_tlstream_timeline_header( -+ TL_STREAM_TYPE_AUX_HEADER, -+ tp_desc_aux, -+ ARRAY_SIZE(tp_desc_aux)); -+ -+ /* Start autoflush timer. */ -+ atomic_set(&autoflush_timer_active, 1); -+ rcode = mod_timer( -+ &autoflush_timer, -+ jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); -+ CSTD_UNUSED(rcode); -+ -+ /* If job dumping is enabled, readjust the software event's -+ * timeout as the default value of 3 seconds is often -+ * insufficient. */ -+ if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { -+ dev_info(kctx->kbdev->dev, -+ "Job dumping is enabled, readjusting the software event's timeout\n"); -+ atomic_set(&kctx->kbdev->js_data.soft_job_timeout_ms, -+ 1800000); -+ } -+ -+ /* Summary stream was cleared during acquire. -+ * Create static timeline objects that will be -+ * read by client. -+ */ -+ kbase_create_timeline_objects(kctx); -+ -+ } else { -+ ret = -EBUSY; -+ } -+ -+ return ret; -+} -+ -+void kbase_tlstream_flush_streams(void) -+{ -+ enum tl_stream_type stype; -+ -+ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) -+ kbasep_tlstream_flush_stream(stype); -+} -+ -+void kbase_tlstream_reset_body_streams(void) -+{ -+ kbasep_timeline_stream_reset( -+ tl_stream[TL_STREAM_TYPE_OBJ]); -+ kbasep_timeline_stream_reset( -+ tl_stream[TL_STREAM_TYPE_AUX]); -+} -+ -+#if MALI_UNIT_TEST -+void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated) -+{ -+ KBASE_DEBUG_ASSERT(bytes_collected); -+ KBASE_DEBUG_ASSERT(bytes_generated); -+ *bytes_collected = atomic_read(&tlstream_bytes_collected); -+ *bytes_generated = atomic_read(&tlstream_bytes_generated); -+} -+#endif /* MALI_UNIT_TEST */ -+ -+/*****************************************************************************/ -+ -+void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid) -+{ -+ const u32 msg_id = KBASE_TL_NEW_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + -+ sizeof(tgid); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &nr, sizeof(nr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &tgid, sizeof(tgid)); -+ -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); -+} -+ -+void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count) -+{ -+ const u32 msg_id = KBASE_TL_NEW_GPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu) + sizeof(id) + -+ sizeof(core_count); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &id, sizeof(id)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &core_count, sizeof(core_count)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); -+} -+ -+void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn) -+{ -+ const u32 msg_id = KBASE_TL_NEW_LPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(nr) + -+ sizeof(fn); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &nr, sizeof(nr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &fn, sizeof(fn)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); -+} -+ -+void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu) -+{ -+ const u32 msg_id = KBASE_TL_LIFELINK_LPU_GPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(lpu) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); -+} -+ -+void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr) -+{ -+ const u32 msg_id = KBASE_TL_NEW_AS; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(nr); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &nr, sizeof(nr)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); -+} -+ -+void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu) -+{ -+ const u32 msg_id = KBASE_TL_LIFELINK_AS_GPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ_SUMMARY, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ_SUMMARY, flags); -+} -+ -+/*****************************************************************************/ -+ -+void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid) -+{ -+ const u32 msg_id = KBASE_TL_NEW_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(nr) + -+ sizeof(tgid); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &nr, sizeof(nr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &tgid, sizeof(tgid)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_new_atom(void *atom, u32 nr) -+{ -+ const u32 msg_id = KBASE_TL_NEW_ATOM; -+ const size_t msg_size = sizeof(msg_id) + sizeof(u64) + sizeof(atom) + -+ sizeof(nr); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &nr, sizeof(nr)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_del_ctx(void *context) -+{ -+ const u32 msg_id = KBASE_TL_DEL_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(context); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_del_atom(void *atom) -+{ -+ const u32 msg_id = KBASE_TL_DEL_ATOM; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu) -+{ -+ const u32 msg_id = KBASE_TL_RET_CTX_LPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context) -+{ -+ const u32 msg_id = KBASE_TL_RET_ATOM_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_ret_atom_lpu( -+ void *atom, void *lpu, const char *attrib_match_list) -+{ -+ const u32 msg_id = KBASE_TL_RET_ATOM_LPU; -+ const size_t msg_s0 = sizeof(u32) + sizeof(char) + -+ strnlen(attrib_match_list, STRLEN_MAX); -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + -+ sizeof(atom) + sizeof(lpu) + msg_s0; -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ pos = kbasep_tlstream_write_string( -+ buffer, pos, attrib_match_list, msg_s0); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu) -+{ -+ const u32 msg_id = KBASE_TL_NRET_CTX_LPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(context) + sizeof(lpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context) -+{ -+ const u32 msg_id = KBASE_TL_NRET_ATOM_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(context); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &context, sizeof(context)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2) -+{ -+ const u32 msg_id = KBASE_TL_DEP_ATOM_ATOM; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom1, sizeof(atom1)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom2, sizeof(atom2)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2) -+{ -+ const u32 msg_id = KBASE_TL_NDEP_ATOM_ATOM; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom1, sizeof(atom1)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom2, sizeof(atom2)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2) -+{ -+ const u32 msg_id = KBASE_TL_RDEP_ATOM_ATOM; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom1) + sizeof(atom2); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom1, sizeof(atom1)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom2, sizeof(atom2)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu) -+{ -+ const u32 msg_id = KBASE_TL_NRET_ATOM_LPU; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(lpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx) -+{ -+ const u32 msg_id = KBASE_TL_RET_AS_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &ctx, sizeof(ctx)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx) -+{ -+ const u32 msg_id = KBASE_TL_NRET_AS_CTX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(as) + sizeof(ctx); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &ctx, sizeof(ctx)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as) -+{ -+ const u32 msg_id = KBASE_TL_RET_ATOM_AS; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as) -+{ -+ const u32 msg_id = KBASE_TL_NRET_ATOM_AS; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(as); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_attrib_atom_config( -+ void *atom, u64 jd, u64 affinity, u32 config) -+{ -+ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_CONFIG; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + -+ sizeof(jd) + sizeof(affinity) + sizeof(config); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &jd, sizeof(jd)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &affinity, sizeof(affinity)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &config, sizeof(config)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio) -+{ -+ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(prio); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &prio, sizeof(prio)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state) -+{ -+ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_STATE; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) + sizeof(state); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &state, sizeof(state)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom) -+{ -+ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_PRIORITY_CHANGE; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_attrib_atom_jit( -+ void *atom, u64 edit_addr, u64 new_addr) -+{ -+ const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom) -+ + sizeof(edit_addr) + sizeof(new_addr); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &edit_addr, sizeof(edit_addr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &new_addr, sizeof(new_addr)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_attrib_as_config( -+ void *as, u64 transtab, u64 memattr, u64 transcfg) -+{ -+ const u32 msg_id = KBASE_TL_ATTRIB_AS_CONFIG; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(as) + -+ sizeof(transtab) + sizeof(memattr) + sizeof(transcfg); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &as, sizeof(as)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &transtab, sizeof(transtab)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &memattr, sizeof(memattr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &transcfg, sizeof(transcfg)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_event_lpu_softstop(void *lpu) -+{ -+ const u32 msg_id = KBASE_TL_EVENT_LPU_SOFTSTOP; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(lpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &lpu, sizeof(lpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom) -+{ -+ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_EX; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom) -+{ -+ const u32 msg_id = KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(atom); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &atom, sizeof(atom)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+void __kbase_tlstream_jd_gpu_soft_reset(void *gpu) -+{ -+ const u32 msg_id = KBASE_JD_GPU_SOFT_RESET; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_OBJ, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_OBJ, flags); -+} -+ -+/*****************************************************************************/ -+ -+void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state) -+{ -+ const u32 msg_id = KBASE_AUX_PM_STATE; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(core_type) + -+ sizeof(state); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &core_type, sizeof(core_type)); -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &state, sizeof(state)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+ -+void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change) -+{ -+ const u32 msg_id = KBASE_AUX_PAGEFAULT; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + -+ sizeof(page_count_change); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, -+ &page_count_change, sizeof(page_count_change)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+ -+void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count) -+{ -+ const u32 msg_id = KBASE_AUX_PAGESALLOC; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(ctx_nr) + -+ sizeof(page_count); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &ctx_nr, sizeof(ctx_nr)); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &page_count, sizeof(page_count)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+ -+void __kbase_tlstream_aux_devfreq_target(u64 target_freq) -+{ -+ const u32 msg_id = KBASE_AUX_DEVFREQ_TARGET; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(target_freq); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &target_freq, sizeof(target_freq)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+ -+void __kbase_tlstream_aux_protected_enter_start(void *gpu) -+{ -+ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_START; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+void __kbase_tlstream_aux_protected_enter_end(void *gpu) -+{ -+ const u32 msg_id = KBASE_AUX_PROTECTED_ENTER_END; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+ -+void __kbase_tlstream_aux_protected_leave_start(void *gpu) -+{ -+ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_START; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -+void __kbase_tlstream_aux_protected_leave_end(void *gpu) -+{ -+ const u32 msg_id = KBASE_AUX_PROTECTED_LEAVE_END; -+ const size_t msg_size = -+ sizeof(msg_id) + sizeof(u64) + sizeof(gpu); -+ unsigned long flags; -+ char *buffer; -+ size_t pos = 0; -+ -+ buffer = kbasep_tlstream_msgbuf_acquire( -+ TL_STREAM_TYPE_AUX, -+ msg_size, &flags); -+ KBASE_DEBUG_ASSERT(buffer); -+ -+ pos = kbasep_tlstream_write_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_tlstream_write_timestamp(buffer, pos); -+ pos = kbasep_tlstream_write_bytes( -+ buffer, pos, &gpu, sizeof(gpu)); -+ KBASE_DEBUG_ASSERT(msg_size == pos); -+ -+ kbasep_tlstream_msgbuf_release(TL_STREAM_TYPE_AUX, flags); -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_tlstream.h b/drivers/gpu/arm/midgard/mali_kbase_tlstream.h -new file mode 100755 -index 000000000..c0a1117d5 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_tlstream.h -@@ -0,0 +1,623 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#if !defined(_KBASE_TLSTREAM_H) -+#define _KBASE_TLSTREAM_H -+ -+#include -+ -+/*****************************************************************************/ -+ -+/** -+ * kbase_tlstream_init - initialize timeline infrastructure in kernel -+ * Return: zero on success, negative number on error -+ */ -+int kbase_tlstream_init(void); -+ -+/** -+ * kbase_tlstream_term - terminate timeline infrastructure in kernel -+ * -+ * Timeline need have to been previously enabled with kbase_tlstream_init(). -+ */ -+void kbase_tlstream_term(void); -+ -+/** -+ * kbase_tlstream_acquire - acquire timeline stream file descriptor -+ * @kctx: kernel common context -+ * @flags: timeline stream flags -+ * -+ * This descriptor is meant to be used by userspace timeline to gain access to -+ * kernel timeline stream. This stream is later broadcasted by user space to the -+ * timeline client. -+ * Only one entity can own the descriptor at any given time. Descriptor shall be -+ * closed if unused. If descriptor cannot be obtained (i.e. when it is already -+ * being used) return will be a negative value. -+ * -+ * Return: file descriptor on success, negative number on error -+ */ -+int kbase_tlstream_acquire(struct kbase_context *kctx, u32 flags); -+ -+/** -+ * kbase_tlstream_flush_streams - flush timeline streams. -+ * -+ * Function will flush pending data in all timeline streams. -+ */ -+void kbase_tlstream_flush_streams(void); -+ -+/** -+ * kbase_tlstream_reset_body_streams - reset timeline body streams. -+ * -+ * Function will discard pending data in all timeline body streams. -+ */ -+void kbase_tlstream_reset_body_streams(void); -+ -+#if MALI_UNIT_TEST -+/** -+ * kbase_tlstream_test - start timeline stream data generator -+ * @tpw_count: number of trace point writers in each context -+ * @msg_delay: time delay in milliseconds between trace points written by one -+ * writer -+ * @msg_count: number of trace points written by one writer -+ * @aux_msg: if non-zero aux messages will be included -+ * -+ * This test starts a requested number of asynchronous writers in both IRQ and -+ * thread context. Each writer will generate required number of test -+ * tracepoints (tracepoints with embedded information about writer that -+ * should be verified by user space reader). Tracepoints will be emitted in -+ * all timeline body streams. If aux_msg is non-zero writer will also -+ * generate not testable tracepoints (tracepoints without information about -+ * writer). These tracepoints are used to check correctness of remaining -+ * timeline message generating functions. Writer will wait requested time -+ * between generating another set of messages. This call blocks until all -+ * writers finish. -+ */ -+void kbase_tlstream_test( -+ unsigned int tpw_count, -+ unsigned int msg_delay, -+ unsigned int msg_count, -+ int aux_msg); -+ -+/** -+ * kbase_tlstream_stats - read timeline stream statistics -+ * @bytes_collected: will hold number of bytes read by the user -+ * @bytes_generated: will hold number of bytes generated by trace points -+ */ -+void kbase_tlstream_stats(u32 *bytes_collected, u32 *bytes_generated); -+#endif /* MALI_UNIT_TEST */ -+ -+/*****************************************************************************/ -+ -+#define TL_ATOM_STATE_IDLE 0 -+#define TL_ATOM_STATE_READY 1 -+#define TL_ATOM_STATE_DONE 2 -+#define TL_ATOM_STATE_POSTED 3 -+ -+void __kbase_tlstream_tl_summary_new_ctx(void *context, u32 nr, u32 tgid); -+void __kbase_tlstream_tl_summary_new_gpu(void *gpu, u32 id, u32 core_count); -+void __kbase_tlstream_tl_summary_new_lpu(void *lpu, u32 nr, u32 fn); -+void __kbase_tlstream_tl_summary_lifelink_lpu_gpu(void *lpu, void *gpu); -+void __kbase_tlstream_tl_summary_new_as(void *as, u32 nr); -+void __kbase_tlstream_tl_summary_lifelink_as_gpu(void *as, void *gpu); -+void __kbase_tlstream_tl_new_ctx(void *context, u32 nr, u32 tgid); -+void __kbase_tlstream_tl_new_atom(void *atom, u32 nr); -+void __kbase_tlstream_tl_del_ctx(void *context); -+void __kbase_tlstream_tl_del_atom(void *atom); -+void __kbase_tlstream_tl_ret_ctx_lpu(void *context, void *lpu); -+void __kbase_tlstream_tl_ret_atom_ctx(void *atom, void *context); -+void __kbase_tlstream_tl_ret_atom_lpu( -+ void *atom, void *lpu, const char *attrib_match_list); -+void __kbase_tlstream_tl_nret_ctx_lpu(void *context, void *lpu); -+void __kbase_tlstream_tl_nret_atom_ctx(void *atom, void *context); -+void __kbase_tlstream_tl_nret_atom_lpu(void *atom, void *lpu); -+void __kbase_tlstream_tl_ret_as_ctx(void *as, void *ctx); -+void __kbase_tlstream_tl_nret_as_ctx(void *as, void *ctx); -+void __kbase_tlstream_tl_ret_atom_as(void *atom, void *as); -+void __kbase_tlstream_tl_nret_atom_as(void *atom, void *as); -+void __kbase_tlstream_tl_dep_atom_atom(void *atom1, void *atom2); -+void __kbase_tlstream_tl_ndep_atom_atom(void *atom1, void *atom2); -+void __kbase_tlstream_tl_rdep_atom_atom(void *atom1, void *atom2); -+void __kbase_tlstream_tl_attrib_atom_config( -+ void *atom, u64 jd, u64 affinity, u32 config); -+void __kbase_tlstream_tl_attrib_atom_priority(void *atom, u32 prio); -+void __kbase_tlstream_tl_attrib_atom_state(void *atom, u32 state); -+void __kbase_tlstream_tl_attrib_atom_priority_change(void *atom); -+void __kbase_tlstream_tl_attrib_atom_jit( -+ void *atom, u64 edit_addr, u64 new_addr); -+void __kbase_tlstream_tl_attrib_as_config( -+ void *as, u64 transtab, u64 memattr, u64 transcfg); -+void __kbase_tlstream_tl_event_atom_softstop_ex(void *atom); -+void __kbase_tlstream_tl_event_lpu_softstop(void *lpu); -+void __kbase_tlstream_tl_event_atom_softstop_issue(void *atom); -+void __kbase_tlstream_jd_gpu_soft_reset(void *gpu); -+void __kbase_tlstream_aux_pm_state(u32 core_type, u64 state); -+void __kbase_tlstream_aux_pagefault(u32 ctx_nr, u64 page_count_change); -+void __kbase_tlstream_aux_pagesalloc(u32 ctx_nr, u64 page_count); -+void __kbase_tlstream_aux_devfreq_target(u64 target_freq); -+void __kbase_tlstream_aux_protected_enter_start(void *gpu); -+void __kbase_tlstream_aux_protected_enter_end(void *gpu); -+void __kbase_tlstream_aux_protected_leave_start(void *gpu); -+void __kbase_tlstream_aux_protected_leave_end(void *gpu); -+ -+#define TLSTREAM_ENABLED (1 << 31) -+ -+extern atomic_t kbase_tlstream_enabled; -+ -+#define __TRACE_IF_ENABLED(trace_name, ...) \ -+ do { \ -+ int enabled = atomic_read(&kbase_tlstream_enabled); \ -+ if (enabled & TLSTREAM_ENABLED) \ -+ __kbase_tlstream_##trace_name(__VA_ARGS__); \ -+ } while (0) -+ -+#define __TRACE_IF_ENABLED_LATENCY(trace_name, ...) \ -+ do { \ -+ int enabled = atomic_read(&kbase_tlstream_enabled); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ -+ __kbase_tlstream_##trace_name(__VA_ARGS__); \ -+ } while (0) -+ -+#define __TRACE_IF_ENABLED_JD(trace_name, ...) \ -+ do { \ -+ int enabled = atomic_read(&kbase_tlstream_enabled); \ -+ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ -+ __kbase_tlstream_##trace_name(__VA_ARGS__); \ -+ } while (0) -+ -+/*****************************************************************************/ -+ -+/** -+ * KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX - create context object in timeline -+ * summary -+ * @context: name of the context object -+ * @nr: context number -+ * @tgid: thread Group Id -+ * -+ * Function emits a timeline message informing about context creation. Context -+ * is created with context number (its attribute), that can be used to link -+ * kbase context with userspace context. -+ * This message is directed to timeline summary stream. -+ */ -+#define KBASE_TLSTREAM_TL_SUMMARY_NEW_CTX(context, nr, tgid) \ -+ __TRACE_IF_ENABLED(tl_summary_new_ctx, context, nr, tgid) -+ -+/** -+ * KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU - create GPU object in timeline summary -+ * @gpu: name of the GPU object -+ * @id: id value of this GPU -+ * @core_count: number of cores this GPU hosts -+ * -+ * Function emits a timeline message informing about GPU creation. GPU is -+ * created with two attributes: id and core count. -+ * This message is directed to timeline summary stream. -+ */ -+#define KBASE_TLSTREAM_TL_SUMMARY_NEW_GPU(gpu, id, core_count) \ -+ __TRACE_IF_ENABLED(tl_summary_new_gpu, gpu, id, core_count) -+ -+/** -+ * KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU - create LPU object in timeline summary -+ * @lpu: name of the Logical Processing Unit object -+ * @nr: sequential number assigned to this LPU -+ * @fn: property describing this LPU's functional abilities -+ * -+ * Function emits a timeline message informing about LPU creation. LPU is -+ * created with two attributes: number linking this LPU with GPU's job slot -+ * and function bearing information about this LPU abilities. -+ * This message is directed to timeline summary stream. -+ */ -+#define KBASE_TLSTREAM_TL_SUMMARY_NEW_LPU(lpu, nr, fn) \ -+ __TRACE_IF_ENABLED(tl_summary_new_lpu, lpu, nr, fn) -+ -+/** -+ * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU - lifelink LPU object to GPU -+ * @lpu: name of the Logical Processing Unit object -+ * @gpu: name of the GPU object -+ * -+ * Function emits a timeline message informing that LPU object shall be deleted -+ * along with GPU object. -+ * This message is directed to timeline summary stream. -+ */ -+#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_LPU_GPU(lpu, gpu) \ -+ __TRACE_IF_ENABLED(tl_summary_lifelink_lpu_gpu, lpu, gpu) -+ -+/** -+ * KBASE_TLSTREAM_TL_SUMMARY_NEW_AS - create address space object in timeline summary -+ * @as: name of the address space object -+ * @nr: sequential number assigned to this address space -+ * -+ * Function emits a timeline message informing about address space creation. -+ * Address space is created with one attribute: number identifying this -+ * address space. -+ * This message is directed to timeline summary stream. -+ */ -+#define KBASE_TLSTREAM_TL_SUMMARY_NEW_AS(as, nr) \ -+ __TRACE_IF_ENABLED(tl_summary_new_as, as, nr) -+ -+/** -+ * KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU - lifelink address space object to GPU -+ * @as: name of the address space object -+ * @gpu: name of the GPU object -+ * -+ * Function emits a timeline message informing that address space object -+ * shall be deleted along with GPU object. -+ * This message is directed to timeline summary stream. -+ */ -+#define KBASE_TLSTREAM_TL_SUMMARY_LIFELINK_AS_GPU(as, gpu) \ -+ __TRACE_IF_ENABLED(tl_summary_lifelink_as_gpu, as, gpu) -+ -+/** -+ * KBASE_TLSTREAM_TL_NEW_CTX - create context object in timeline -+ * @context: name of the context object -+ * @nr: context number -+ * @tgid: thread Group Id -+ * -+ * Function emits a timeline message informing about context creation. Context -+ * is created with context number (its attribute), that can be used to link -+ * kbase context with userspace context. -+ */ -+#define KBASE_TLSTREAM_TL_NEW_CTX(context, nr, tgid) \ -+ __TRACE_IF_ENABLED(tl_new_ctx, context, nr, tgid) -+ -+/** -+ * KBASE_TLSTREAM_TL_NEW_ATOM - create atom object in timeline -+ * @atom: name of the atom object -+ * @nr: sequential number assigned to this atom -+ * -+ * Function emits a timeline message informing about atom creation. Atom is -+ * created with atom number (its attribute) that links it with actual work -+ * bucket id understood by hardware. -+ */ -+#define KBASE_TLSTREAM_TL_NEW_ATOM(atom, nr) \ -+ __TRACE_IF_ENABLED(tl_new_atom, atom, nr) -+ -+/** -+ * KBASE_TLSTREAM_TL_DEL_CTX - destroy context object in timeline -+ * @context: name of the context object -+ * -+ * Function emits a timeline message informing that context object ceased to -+ * exist. -+ */ -+#define KBASE_TLSTREAM_TL_DEL_CTX(context) \ -+ __TRACE_IF_ENABLED(tl_del_ctx, context) -+ -+/** -+ * KBASE_TLSTREAM_TL_DEL_ATOM - destroy atom object in timeline -+ * @atom: name of the atom object -+ * -+ * Function emits a timeline message informing that atom object ceased to -+ * exist. -+ */ -+#define KBASE_TLSTREAM_TL_DEL_ATOM(atom) \ -+ __TRACE_IF_ENABLED(tl_del_atom, atom) -+ -+/** -+ * KBASE_TLSTREAM_TL_RET_CTX_LPU - retain context by LPU -+ * @context: name of the context object -+ * @lpu: name of the Logical Processing Unit object -+ * -+ * Function emits a timeline message informing that context is being held -+ * by LPU and must not be deleted unless it is released. -+ */ -+#define KBASE_TLSTREAM_TL_RET_CTX_LPU(context, lpu) \ -+ __TRACE_IF_ENABLED(tl_ret_ctx_lpu, context, lpu) -+ -+/** -+ * KBASE_TLSTREAM_TL_RET_ATOM_CTX - retain atom by context -+ * @atom: name of the atom object -+ * @context: name of the context object -+ * -+ * Function emits a timeline message informing that atom object is being held -+ * by context and must not be deleted unless it is released. -+ */ -+#define KBASE_TLSTREAM_TL_RET_ATOM_CTX(atom, context) \ -+ __TRACE_IF_ENABLED(tl_ret_atom_ctx, atom, context) -+ -+/** -+ * KBASE_TLSTREAM_TL_RET_ATOM_LPU - retain atom by LPU -+ * @atom: name of the atom object -+ * @lpu: name of the Logical Processing Unit object -+ * @attrib_match_list: list containing match operator attributes -+ * -+ * Function emits a timeline message informing that atom object is being held -+ * by LPU and must not be deleted unless it is released. -+ */ -+#define KBASE_TLSTREAM_TL_RET_ATOM_LPU(atom, lpu, attrib_match_list) \ -+ __TRACE_IF_ENABLED(tl_ret_atom_lpu, atom, lpu, attrib_match_list) -+ -+/** -+ * KBASE_TLSTREAM_TL_NRET_CTX_LPU - release context by LPU -+ * @context: name of the context object -+ * @lpu: name of the Logical Processing Unit object -+ * -+ * Function emits a timeline message informing that context is being released -+ * by LPU object. -+ */ -+#define KBASE_TLSTREAM_TL_NRET_CTX_LPU(context, lpu) \ -+ __TRACE_IF_ENABLED(tl_nret_ctx_lpu, context, lpu) -+ -+/** -+ * KBASE_TLSTREAM_TL_NRET_ATOM_CTX - release atom by context -+ * @atom: name of the atom object -+ * @context: name of the context object -+ * -+ * Function emits a timeline message informing that atom object is being -+ * released by context. -+ */ -+#define KBASE_TLSTREAM_TL_NRET_ATOM_CTX(atom, context) \ -+ __TRACE_IF_ENABLED(tl_nret_atom_ctx, atom, context) -+ -+/** -+ * KBASE_TLSTREAM_TL_NRET_ATOM_LPU - release atom by LPU -+ * @atom: name of the atom object -+ * @lpu: name of the Logical Processing Unit object -+ * -+ * Function emits a timeline message informing that atom object is being -+ * released by LPU. -+ */ -+#define KBASE_TLSTREAM_TL_NRET_ATOM_LPU(atom, lpu) \ -+ __TRACE_IF_ENABLED(tl_nret_atom_lpu, atom, lpu) -+ -+/** -+ * KBASE_TLSTREAM_TL_RET_AS_CTX - lifelink address space object to context -+ * @as: name of the address space object -+ * @ctx: name of the context object -+ * -+ * Function emits a timeline message informing that address space object -+ * is being held by the context object. -+ */ -+#define KBASE_TLSTREAM_TL_RET_AS_CTX(as, ctx) \ -+ __TRACE_IF_ENABLED(tl_ret_as_ctx, as, ctx) -+ -+/** -+ * KBASE_TLSTREAM_TL_NRET_AS_CTX - release address space by context -+ * @as: name of the address space object -+ * @ctx: name of the context object -+ * -+ * Function emits a timeline message informing that address space object -+ * is being released by atom. -+ */ -+#define KBASE_TLSTREAM_TL_NRET_AS_CTX(as, ctx) \ -+ __TRACE_IF_ENABLED(tl_nret_as_ctx, as, ctx) -+ -+/** -+ * KBASE_TLSTREAM_TL_RET_ATOM_AS - retain atom by address space -+ * @atom: name of the atom object -+ * @as: name of the address space object -+ * -+ * Function emits a timeline message informing that atom object is being held -+ * by address space and must not be deleted unless it is released. -+ */ -+#define KBASE_TLSTREAM_TL_RET_ATOM_AS(atom, as) \ -+ __TRACE_IF_ENABLED(tl_ret_atom_as, atom, as) -+ -+/** -+ * KBASE_TLSTREAM_TL_NRET_ATOM_AS - release atom by address space -+ * @atom: name of the atom object -+ * @as: name of the address space object -+ * -+ * Function emits a timeline message informing that atom object is being -+ * released by address space. -+ */ -+#define KBASE_TLSTREAM_TL_NRET_ATOM_AS(atom, as) \ -+ __TRACE_IF_ENABLED(tl_nret_atom_as, atom, as) -+ -+/** -+ * KBASE_TLSTREAM_TL_DEP_ATOM_ATOM - parent atom depends on child atom -+ * @atom1: name of the child atom object -+ * @atom2: name of the parent atom object that depends on child atom -+ * -+ * Function emits a timeline message informing that parent atom waits for -+ * child atom object to be completed before start its execution. -+ */ -+#define KBASE_TLSTREAM_TL_DEP_ATOM_ATOM(atom1, atom2) \ -+ __TRACE_IF_ENABLED(tl_dep_atom_atom, atom1, atom2) -+ -+/** -+ * KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM - dependency between atoms resolved -+ * @atom1: name of the child atom object -+ * @atom2: name of the parent atom object that depended on child atom -+ * -+ * Function emits a timeline message informing that parent atom execution -+ * dependency on child atom has been resolved. -+ */ -+#define KBASE_TLSTREAM_TL_NDEP_ATOM_ATOM(atom1, atom2) \ -+ __TRACE_IF_ENABLED(tl_ndep_atom_atom, atom1, atom2) -+ -+/** -+ * KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM - information about already resolved dependency between atoms -+ * @atom1: name of the child atom object -+ * @atom2: name of the parent atom object that depended on child atom -+ * -+ * Function emits a timeline message informing that parent atom execution -+ * dependency on child atom has been resolved. -+ */ -+#define KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM(atom1, atom2) \ -+ __TRACE_IF_ENABLED(tl_rdep_atom_atom, atom1, atom2) -+ -+/** -+ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG - atom job slot attributes -+ * @atom: name of the atom object -+ * @jd: job descriptor address -+ * @affinity: job affinity -+ * @config: job config -+ * -+ * Function emits a timeline message containing atom attributes. -+ */ -+#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(atom, jd, affinity, config) \ -+ __TRACE_IF_ENABLED(tl_attrib_atom_config, atom, jd, affinity, config) -+ -+/** -+ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY - atom priority -+ * @atom: name of the atom object -+ * @prio: atom priority -+ * -+ * Function emits a timeline message containing atom priority. -+ */ -+#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(atom, prio) \ -+ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority, atom, prio) -+ -+/** -+ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE - atom state -+ * @atom: name of the atom object -+ * @state: atom state -+ * -+ * Function emits a timeline message containing atom state. -+ */ -+#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(atom, state) \ -+ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_state, atom, state) -+ -+/** -+ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE - atom caused priority change -+ * @atom: name of the atom object -+ * -+ * Function emits a timeline message signalling priority change -+ */ -+#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY_CHANGE(atom) \ -+ __TRACE_IF_ENABLED_LATENCY(tl_attrib_atom_priority_change, atom) -+ -+/** -+ * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit happened on atom -+ * @atom: atom identifier -+ * @edit_addr: address edited by jit -+ * @new_addr: address placed into the edited location -+ */ -+#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(atom, edit_addr, new_addr) \ -+ __TRACE_IF_ENABLED_JD(tl_attrib_atom_jit, atom, edit_addr, new_addr) -+ -+/** -+ * KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG - address space attributes -+ * @as: assigned address space -+ * @transtab: configuration of the TRANSTAB register -+ * @memattr: configuration of the MEMATTR register -+ * @transcfg: configuration of the TRANSCFG register (or zero if not present) -+ * -+ * Function emits a timeline message containing address space attributes. -+ */ -+#define KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(as, transtab, memattr, transcfg) \ -+ __TRACE_IF_ENABLED(tl_attrib_as_config, as, transtab, memattr, transcfg) -+ -+/** -+ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ex -+ * @atom: atom identifier -+ */ -+#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(atom) \ -+ __TRACE_IF_ENABLED(tl_event_atom_softstop_ex, atom) -+ -+/** -+ * KBASE_TLSTREAM_TL_EVENT_LPU_softstop -+ * @lpu: name of the LPU object -+ */ -+#define KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(lpu) \ -+ __TRACE_IF_ENABLED(tl_event_lpu_softstop, lpu) -+ -+/** -+ * KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_issue -+ * @atom: atom identifier -+ */ -+#define KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(atom) \ -+ __TRACE_IF_ENABLED(tl_event_atom_softstop_issue, atom) -+ -+/** -+ * KBASE_TLSTREAM_JD_GPU_SOFT_RESET - The GPU is being soft reset -+ * @gpu: name of the GPU object -+ * -+ * This imperative tracepoint is specific to job dumping. -+ * Function emits a timeline message indicating GPU soft reset. -+ */ -+#define KBASE_TLSTREAM_JD_GPU_SOFT_RESET(gpu) \ -+ __TRACE_IF_ENABLED(jd_gpu_soft_reset, gpu) -+ -+ -+/** -+ * KBASE_TLSTREAM_AUX_PM_STATE - timeline message: power management state -+ * @core_type: core type (shader, tiler, l2 cache, l3 cache) -+ * @state: 64bits bitmask reporting power state of the cores (1-ON, 0-OFF) -+ */ -+#define KBASE_TLSTREAM_AUX_PM_STATE(core_type, state) \ -+ __TRACE_IF_ENABLED(aux_pm_state, core_type, state) -+ -+/** -+ * KBASE_TLSTREAM_AUX_PAGEFAULT - timeline message: MMU page fault event -+ * resulting in new pages being mapped -+ * @ctx_nr: kernel context number -+ * @page_count_change: number of pages to be added -+ */ -+#define KBASE_TLSTREAM_AUX_PAGEFAULT(ctx_nr, page_count_change) \ -+ __TRACE_IF_ENABLED(aux_pagefault, ctx_nr, page_count_change) -+ -+/** -+ * KBASE_TLSTREAM_AUX_PAGESALLOC - timeline message: total number of allocated -+ * pages is changed -+ * @ctx_nr: kernel context number -+ * @page_count: number of pages used by the context -+ */ -+#define KBASE_TLSTREAM_AUX_PAGESALLOC(ctx_nr, page_count) \ -+ __TRACE_IF_ENABLED(aux_pagesalloc, ctx_nr, page_count) -+ -+/** -+ * KBASE_TLSTREAM_AUX_DEVFREQ_TARGET - timeline message: new target DVFS -+ * frequency -+ * @target_freq: new target frequency -+ */ -+#define KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(target_freq) \ -+ __TRACE_IF_ENABLED(aux_devfreq_target, target_freq) -+ -+/** -+ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START - The GPU has started transitioning -+ * to protected mode -+ * @gpu: name of the GPU object -+ * -+ * Function emits a timeline message indicating the GPU is starting to -+ * transition to protected mode. -+ */ -+#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(gpu) \ -+ __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_start, gpu) -+ -+/** -+ * KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END - The GPU has finished transitioning -+ * to protected mode -+ * @gpu: name of the GPU object -+ * -+ * Function emits a timeline message indicating the GPU has finished -+ * transitioning to protected mode. -+ */ -+#define KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(gpu) \ -+ __TRACE_IF_ENABLED_LATENCY(aux_protected_enter_end, gpu) -+ -+/** -+ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START - The GPU has started transitioning -+ * to non-protected mode -+ * @gpu: name of the GPU object -+ * -+ * Function emits a timeline message indicating the GPU is starting to -+ * transition to non-protected mode. -+ */ -+#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(gpu) \ -+ __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_start, gpu) -+ -+/** -+ * KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END - The GPU has finished transitioning -+ * to non-protected mode -+ * @gpu: name of the GPU object -+ * -+ * Function emits a timeline message indicating the GPU has finished -+ * transitioning to non-protected mode. -+ */ -+#define KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(gpu) \ -+ __TRACE_IF_ENABLED_LATENCY(aux_protected_leave_end, gpu) -+ -+#endif /* _KBASE_TLSTREAM_H */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h b/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h -new file mode 100755 -index 000000000..e2e054420 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_trace_defs.h -@@ -0,0 +1,264 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** -+ * ***** DO NOT INCLUDE DIRECTLY ***** -+ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ -+ -+/* -+ * The purpose of this header file is just to contain a list of trace code idenitifers -+ * -+ * Each identifier is wrapped in a macro, so that its string form and enum form can be created -+ * -+ * Each macro is separated with a comma, to allow insertion into an array initializer or enum definition block. -+ * -+ * This allows automatic creation of an enum and a corresponding array of strings -+ * -+ * Before #including, the includer MUST #define KBASE_TRACE_CODE_MAKE_CODE. -+ * After #including, the includer MUST #under KBASE_TRACE_CODE_MAKE_CODE. -+ * -+ * e.g.: -+ * #define KBASE_TRACE_CODE( X ) KBASE_TRACE_CODE_ ## X -+ * typedef enum -+ * { -+ * #define KBASE_TRACE_CODE_MAKE_CODE( X ) KBASE_TRACE_CODE( X ) -+ * #include "mali_kbase_trace_defs.h" -+ * #undef KBASE_TRACE_CODE_MAKE_CODE -+ * } kbase_trace_code; -+ * -+ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE -+ * -+ * -+ * The use of the macro here is: -+ * - KBASE_TRACE_CODE_MAKE_CODE( X ) -+ * -+ * Which produces: -+ * - For an enum, KBASE_TRACE_CODE_X -+ * - For a string, "X" -+ * -+ * -+ * For example: -+ * - KBASE_TRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to: -+ * - KBASE_TRACE_CODE_JM_JOB_COMPLETE for the enum -+ * - "JM_JOB_COMPLETE" for the string -+ * - To use it to trace an event, do: -+ * - KBASE_TRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val ); -+ */ -+ -+#if 0 /* Dummy section to avoid breaking formatting */ -+int dummy_array[] = { -+#endif -+ -+/* -+ * Core events -+ */ -+ /* no info_val, no gpu_addr, no atom */ -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), -+ /* no info_val, no gpu_addr, no atom */ -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM), -+ /* info_val == GPU_IRQ_STATUS register */ -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ), -+ /* info_val == bits cleared */ -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR), -+ /* info_val == GPU_IRQ_STATUS register */ -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE), -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET), -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET), -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR), -+ /* GPU addr==dump address */ -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE), -+ KBASE_TRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES), -+/* -+ * Job Slot management events -+ */ -+ /* info_val==irq rawstat at start */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ), -+ /* info_val==jobs processed */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_IRQ_END), -+/* In the following: -+ * -+ * - ctx is set if a corresponding job found (NULL otherwise, e.g. some soft-stop cases) -+ * - uatom==kernel-side mapped uatom address (for correlation with user-side) -+ */ -+ /* info_val==exit code; gpu_addr==chain gpuaddr */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_JOB_DONE), -+ /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT), -+ /* gpu_addr is as follows: -+ * - If JS_STATUS active after soft-stop, val==gpu addr written to -+ * JS_HEAD on submit -+ * - otherwise gpu_addr==0 */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), -+ /* gpu_addr==JS_HEAD read */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP), -+ /* gpu_addr==JS_HEAD read */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), -+ /* gpu_addr==JS_HEAD read */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), -+ /* gpu_addr==JS_TAIL read */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), -+/* gpu_addr is as follows: -+ * - If JS_STATUS active before soft-stop, val==JS_HEAD -+ * - otherwise gpu_addr==0 -+ */ -+ /* gpu_addr==JS_HEAD read */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), -+ /* info_val == is_scheduled */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), -+ /* info_val == is_scheduled */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_ZAP_DONE), -+ /* info_val == nr jobs submitted */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), -+ /* gpu_addr==JS_HEAD_NEXT last written */ -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), -+ KBASE_TRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), -+/* -+ * Job dispatch events -+ */ -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), -+ /* gpu_addr==0, info_val==0, uatom==0 */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), -+/* -+ * Scheduler Core events -+ */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX_NOLOCK), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_ADD_JOB), -+ /* gpu_addr==last value written/would be written to JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_RETAIN_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_RELEASE_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), -+ /* gpu_addr==value to write into JS_HEAD */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), -+ /* kctx is the one being evicted, info_val == kctx to put in */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_FAST_START_EVICTS_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), -+ /* info_val == lower 32 bits of affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), -+ /* info_val == lower 32 bits of affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), -+ /* info_val == lower 32 bits of affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), -+ /* info_val == lower 32 bits of rechecked affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), -+ /* info_val == lower 32 bits of rechecked affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), -+ /* info_val == lower 32 bits of affinity */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), -+ /* info_val == the ctx attribute now on ctx */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), -+ /* info_val == the ctx attribute now on runpool */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), -+ /* info_val == the ctx attribute now off ctx */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), -+ /* info_val == the ctx attribute now off runpool */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), -+/* -+ * Scheduler Policy events -+ */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), -+ /* info_val == whether it was evicted */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), -+ /* gpu_addr==JS_HEAD to write if the job were run */ -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), -+ KBASE_TRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), -+/* -+ * Power Management Events -+ */ -+ KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_TILER), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_PWRON_L2), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_TILER), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_PWROFF_L2), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), -+ /* PM_DESIRED_REACHED: gpu_addr == pm.gpu_in_desired_state */ -+ KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_INUSE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_INUSE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_SHADER_NEEDED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_REGISTER_CHANGE_TILER_NEEDED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_INUSE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_INUSE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_SHADER_NEEDED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_UNREQUEST_CHANGE_TILER_NEEDED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_ON), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_GPU_OFF), -+ /* info_val == policy number, or -1 for "Already changing" */ -+ KBASE_TRACE_CODE_MAKE_CODE(PM_SET_POLICY), -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY), -+ /* info_val == policy number */ -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT), -+ /* info_val == policy number */ -+ KBASE_TRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM), -+/* Unused code just to make it easier to not have a comma at the end. -+ * All other codes MUST come before this */ -+ KBASE_TRACE_CODE_MAKE_CODE(DUMMY) -+ -+#if 0 /* Dummy section to avoid breaking formatting */ -+}; -+#endif -+ -+/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c -new file mode 100755 -index 000000000..5830e87f0 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.c -@@ -0,0 +1,236 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+#include -+#include -+ -+#define CREATE_TRACE_POINTS -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+#include "mali_timeline.h" -+ -+#include -+#include -+ -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atoms_in_flight); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_atom); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_active); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_slot_action); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_gpu_power_active); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_l2_power_active); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_event); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_slot_atom); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_pm_checktrans); -+EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_context_active); -+ -+struct kbase_trace_timeline_desc { -+ char *enum_str; -+ char *desc; -+ char *format; -+ char *format_desc; -+}; -+ -+static struct kbase_trace_timeline_desc kbase_trace_timeline_desc_table[] = { -+ #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) { #enum_val, desc, format, format_desc } -+ #include "mali_kbase_trace_timeline_defs.h" -+ #undef KBASE_TIMELINE_TRACE_CODE -+}; -+ -+#define KBASE_NR_TRACE_CODES ARRAY_SIZE(kbase_trace_timeline_desc_table) -+ -+static void *kbasep_trace_timeline_seq_start(struct seq_file *s, loff_t *pos) -+{ -+ if (*pos >= KBASE_NR_TRACE_CODES) -+ return NULL; -+ -+ return &kbase_trace_timeline_desc_table[*pos]; -+} -+ -+static void kbasep_trace_timeline_seq_stop(struct seq_file *s, void *data) -+{ -+} -+ -+static void *kbasep_trace_timeline_seq_next(struct seq_file *s, void *data, loff_t *pos) -+{ -+ (*pos)++; -+ -+ if (*pos == KBASE_NR_TRACE_CODES) -+ return NULL; -+ -+ return &kbase_trace_timeline_desc_table[*pos]; -+} -+ -+static int kbasep_trace_timeline_seq_show(struct seq_file *s, void *data) -+{ -+ struct kbase_trace_timeline_desc *trace_desc = data; -+ -+ seq_printf(s, "%s#%s#%s#%s\n", trace_desc->enum_str, trace_desc->desc, trace_desc->format, trace_desc->format_desc); -+ return 0; -+} -+ -+ -+static const struct seq_operations kbasep_trace_timeline_seq_ops = { -+ .start = kbasep_trace_timeline_seq_start, -+ .next = kbasep_trace_timeline_seq_next, -+ .stop = kbasep_trace_timeline_seq_stop, -+ .show = kbasep_trace_timeline_seq_show, -+}; -+ -+static int kbasep_trace_timeline_debugfs_open(struct inode *inode, struct file *file) -+{ -+ return seq_open(file, &kbasep_trace_timeline_seq_ops); -+} -+ -+static const struct file_operations kbasep_trace_timeline_debugfs_fops = { -+ .open = kbasep_trace_timeline_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+#ifdef CONFIG_DEBUG_FS -+ -+void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("mali_timeline_defs", -+ S_IRUGO, kbdev->mali_debugfs_directory, NULL, -+ &kbasep_trace_timeline_debugfs_fops); -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (kbdev->timeline.slot_atoms_submitted[js] > 0) { -+ KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 1); -+ } else { -+ base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); -+ -+ KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 1); -+ KBASE_TIMELINE_JOB_START(kctx, js, atom_number); -+ } -+ ++kbdev->timeline.slot_atoms_submitted[js]; -+ -+ KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); -+} -+ -+void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js, -+ kbasep_js_atom_done_code done_code) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) { -+ KBASE_TIMELINE_JOB_START_NEXT(kctx, js, 0); -+ } else { -+ /* Job finished in JS_HEAD */ -+ base_atom_id atom_number = kbase_jd_atom_id(kctx, katom); -+ -+ KBASE_TIMELINE_JOB_START_HEAD(kctx, js, 0); -+ KBASE_TIMELINE_JOB_STOP(kctx, js, atom_number); -+ -+ /* see if we need to trace the job in JS_NEXT moving to JS_HEAD */ -+ if (kbase_backend_nr_atoms_submitted(kbdev, js)) { -+ struct kbase_jd_atom *next_katom; -+ struct kbase_context *next_kctx; -+ -+ /* Peek the next atom - note that the atom in JS_HEAD will already -+ * have been dequeued */ -+ next_katom = kbase_backend_inspect_head(kbdev, js); -+ WARN_ON(!next_katom); -+ next_kctx = next_katom->kctx; -+ KBASE_TIMELINE_JOB_START_NEXT(next_kctx, js, 0); -+ KBASE_TIMELINE_JOB_START_HEAD(next_kctx, js, 1); -+ KBASE_TIMELINE_JOB_START(next_kctx, js, kbase_jd_atom_id(next_kctx, next_katom)); -+ } -+ } -+ -+ --kbdev->timeline.slot_atoms_submitted[js]; -+ -+ KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, kbdev->timeline.slot_atoms_submitted[js]); -+} -+ -+void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) -+{ -+ int uid = 0; -+ int old_uid; -+ -+ /* If a producer already exists for the event, try to use their UID (multiple-producers) */ -+ uid = atomic_read(&kbdev->timeline.pm_event_uid[event_sent]); -+ old_uid = uid; -+ -+ /* Get a new non-zero UID if we don't have one yet */ -+ while (!uid) -+ uid = atomic_inc_return(&kbdev->timeline.pm_event_uid_counter); -+ -+ /* Try to use this UID */ -+ if (old_uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event_sent], old_uid, uid)) -+ /* If it changed, raced with another producer: we've lost this UID */ -+ uid = 0; -+ -+ KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_sent, uid); -+} -+ -+void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) -+{ -+ int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); -+ -+ if (uid != 0) { -+ if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) -+ /* If it changed, raced with another consumer: we've lost this UID */ -+ uid = 0; -+ -+ KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); -+ } -+} -+ -+void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) -+{ -+ int uid = atomic_read(&kbdev->timeline.pm_event_uid[event]); -+ -+ if (uid != atomic_cmpxchg(&kbdev->timeline.pm_event_uid[event], uid, 0)) -+ /* If it changed, raced with another consumer: we've lost this UID */ -+ uid = 0; -+ -+ KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event, uid); -+} -+ -+void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ /* Simply log the start of the transition */ -+ kbdev->timeline.l2_transitioning = true; -+ KBASE_TIMELINE_POWERING_L2(kbdev); -+} -+ -+void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ /* Simply log the end of the transition */ -+ if (kbdev->timeline.l2_transitioning) { -+ kbdev->timeline.l2_transitioning = false; -+ KBASE_TIMELINE_POWERED_L2(kbdev); -+ } -+} -+ -+#endif /* CONFIG_MALI_TRACE_TIMELINE */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h -new file mode 100755 -index 000000000..a04f7c142 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline.h -@@ -0,0 +1,363 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#if !defined(_KBASE_TRACE_TIMELINE_H) -+#define _KBASE_TRACE_TIMELINE_H -+ -+#ifdef CONFIG_MALI_TRACE_TIMELINE -+ -+enum kbase_trace_timeline_code { -+ #define KBASE_TIMELINE_TRACE_CODE(enum_val, desc, format, format_desc) enum_val -+ #include "mali_kbase_trace_timeline_defs.h" -+ #undef KBASE_TIMELINE_TRACE_CODE -+}; -+ -+#ifdef CONFIG_DEBUG_FS -+ -+/** Initialize Timeline DebugFS entries */ -+void kbasep_trace_timeline_debugfs_init(struct kbase_device *kbdev); -+ -+#else /* CONFIG_DEBUG_FS */ -+ -+#define kbasep_trace_timeline_debugfs_init CSTD_NOP -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -+/* mali_timeline.h defines kernel tracepoints used by the KBASE_TIMELINE -+ * functions. -+ * Output is timestamped by either sched_clock() (default), local_clock(), or -+ * cpu_clock(), depending on /sys/kernel/debug/tracing/trace_clock */ -+#include "mali_timeline.h" -+ -+/* Trace number of atoms in flight for kctx (atoms either not completed, or in -+ process of being returned to user */ -+#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_atoms_in_flight(ts.tv_sec, ts.tv_nsec, \ -+ (int)kctx->timeline.owner_tgid, \ -+ count); \ -+ } while (0) -+ -+/* Trace atom_id being Ready to Run */ -+#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_atom(ts.tv_sec, ts.tv_nsec, \ -+ CTX_FLOW_ATOM_READY, \ -+ (int)kctx->timeline.owner_tgid, \ -+ atom_id); \ -+ } while (0) -+ -+/* Trace number of atoms submitted to job slot js -+ * -+ * NOTE: This uses a different tracepoint to the head/next/soft-stop actions, -+ * so that those actions can be filtered out separately from this -+ * -+ * This is because this is more useful, as we can use it to calculate general -+ * utilization easily and accurately */ -+#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_slot_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_SLOT_ACTIVE, \ -+ (int)kctx->timeline.owner_tgid, \ -+ js, count); \ -+ } while (0) -+ -+ -+/* Trace atoms present in JS_NEXT */ -+#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_SLOT_NEXT, \ -+ (int)kctx->timeline.owner_tgid, \ -+ js, count); \ -+ } while (0) -+ -+/* Trace atoms present in JS_HEAD */ -+#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_SLOT_HEAD, \ -+ (int)kctx->timeline.owner_tgid, \ -+ js, count); \ -+ } while (0) -+ -+/* Trace that a soft stop/evict from next is being attempted on a slot */ -+#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_slot_action(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_SLOT_STOPPING, \ -+ (kctx) ? (int)kctx->timeline.owner_tgid : 0, \ -+ js, count); \ -+ } while (0) -+ -+ -+ -+/* Trace state of overall GPU power */ -+#define KBASE_TIMELINE_GPU_POWER(kbdev, active) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_POWER_ACTIVE, active); \ -+ } while (0) -+ -+/* Trace state of tiler power */ -+#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_POWER_TILER_ACTIVE, \ -+ hweight64(bitmap)); \ -+ } while (0) -+ -+/* Trace number of shaders currently powered */ -+#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_POWER_SHADER_ACTIVE, \ -+ hweight64(bitmap)); \ -+ } while (0) -+ -+/* Trace state of L2 power */ -+#define KBASE_TIMELINE_POWER_L2(kbdev, bitmap) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_gpu_power_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_SET_GPU_POWER_L2_ACTIVE, \ -+ hweight64(bitmap)); \ -+ } while (0) -+ -+/* Trace state of L2 cache*/ -+#define KBASE_TIMELINE_POWERING_L2(kbdev) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_FLOW_GPU_POWER_L2_POWERING, \ -+ 1); \ -+ } while (0) -+ -+#define KBASE_TIMELINE_POWERED_L2(kbdev) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_l2_power_active(ts.tv_sec, ts.tv_nsec, \ -+ SW_FLOW_GPU_POWER_L2_ACTIVE, \ -+ 1); \ -+ } while (0) -+ -+/* Trace kbase_pm_send_event message send */ -+#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ -+ SW_FLOW_PM_SEND_EVENT, \ -+ event_type, pm_event_id); \ -+ } while (0) -+ -+/* Trace kbase_pm_worker message receive */ -+#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_pm_event(ts.tv_sec, ts.tv_nsec, \ -+ SW_FLOW_PM_HANDLE_EVENT, \ -+ event_type, pm_event_id); \ -+ } while (0) -+ -+ -+/* Trace atom_id starting in JS_HEAD */ -+#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ -+ HW_START_GPU_JOB_CHAIN_SW_APPROX, \ -+ (int)kctx->timeline.owner_tgid, \ -+ js, _consumerof_atom_number); \ -+ } while (0) -+ -+/* Trace atom_id stopping on JS_HEAD */ -+#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_slot_atom(ts.tv_sec, ts.tv_nsec, \ -+ HW_STOP_GPU_JOB_CHAIN_SW_APPROX, \ -+ (int)kctx->timeline.owner_tgid, \ -+ js, _producerof_atom_number_completed); \ -+ } while (0) -+ -+/** Trace beginning/end of a call to kbase_pm_check_transitions_nolock from a -+ * certin caller */ -+#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_pm_checktrans(ts.tv_sec, ts.tv_nsec, \ -+ trace_code, 1); \ -+ } while (0) -+ -+/* Trace number of contexts active */ -+#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) \ -+ do { \ -+ struct timespec64 ts; \ -+ ktime_get_raw_ts64(&ts); \ -+ trace_mali_timeline_context_active(ts.tv_sec, ts.tv_nsec, \ -+ count); \ -+ } while (0) -+ -+/* NOTE: kbase_timeline_pm_cores_func() is in mali_kbase_pm_policy.c */ -+ -+/** -+ * Trace that an atom is starting on a job slot -+ * -+ * The caller must be holding hwaccess_lock -+ */ -+void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js); -+ -+/** -+ * Trace that an atom has done on a job slot -+ * -+ * 'Done' in this sense can occur either because: -+ * - the atom in JS_HEAD finished -+ * - the atom in JS_NEXT was evicted -+ * -+ * Whether the atom finished or was evicted is passed in @a done_code -+ * -+ * It is assumed that the atom has already been removed from the submit slot, -+ * with either: -+ * - kbasep_jm_dequeue_submit_slot() -+ * - kbasep_jm_dequeue_tail_submit_slot() -+ * -+ * The caller must be holding hwaccess_lock -+ */ -+void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js, -+ kbasep_js_atom_done_code done_code); -+ -+ -+/** Trace a pm event starting */ -+void kbase_timeline_pm_send_event(struct kbase_device *kbdev, -+ enum kbase_timeline_pm_event event_sent); -+ -+/** Trace a pm event finishing */ -+void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); -+ -+/** Check whether a pm event was present, and if so trace finishing it */ -+void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event); -+ -+/** Trace L2 power-up start */ -+void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev); -+ -+/** Trace L2 power-up done */ -+void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev); -+ -+#else -+ -+#define KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, count) CSTD_NOP() -+ -+#define KBASE_TIMELINE_ATOM_READY(kctx, atom_id) CSTD_NOP() -+ -+#define KBASE_TIMELINE_ATOMS_SUBMITTED(kctx, js, count) CSTD_NOP() -+ -+#define KBASE_TIMELINE_JOB_START_NEXT(kctx, js, count) CSTD_NOP() -+ -+#define KBASE_TIMELINE_JOB_START_HEAD(kctx, js, count) CSTD_NOP() -+ -+#define KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, count) CSTD_NOP() -+ -+#define KBASE_TIMELINE_GPU_POWER(kbdev, active) CSTD_NOP() -+ -+#define KBASE_TIMELINE_POWER_TILER(kbdev, bitmap) CSTD_NOP() -+ -+#define KBASE_TIMELINE_POWER_SHADER(kbdev, bitmap) CSTD_NOP() -+ -+#define KBASE_TIMELINE_POWER_L2(kbdev, active) CSTD_NOP() -+ -+#define KBASE_TIMELINE_POWERING_L2(kbdev) CSTD_NOP() -+ -+#define KBASE_TIMELINE_POWERED_L2(kbdev) CSTD_NOP() -+ -+#define KBASE_TIMELINE_PM_SEND_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() -+ -+#define KBASE_TIMELINE_PM_HANDLE_EVENT(kbdev, event_type, pm_event_id) CSTD_NOP() -+ -+#define KBASE_TIMELINE_JOB_START(kctx, js, _consumerof_atom_number) CSTD_NOP() -+ -+#define KBASE_TIMELINE_JOB_STOP(kctx, js, _producerof_atom_number_completed) CSTD_NOP() -+ -+#define KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code) CSTD_NOP() -+ -+#define KBASE_TIMELINE_CONTEXT_ACTIVE(kbdev, count) CSTD_NOP() -+ -+static inline void kbase_timeline_job_slot_submit(struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+} -+ -+static inline void kbase_timeline_job_slot_done(struct kbase_device *kbdev, struct kbase_context *kctx, -+ struct kbase_jd_atom *katom, int js, -+ kbasep_js_atom_done_code done_code) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+} -+ -+static inline void kbase_timeline_pm_send_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event_sent) -+{ -+} -+ -+static inline void kbase_timeline_pm_check_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) -+{ -+} -+ -+static inline void kbase_timeline_pm_handle_event(struct kbase_device *kbdev, enum kbase_timeline_pm_event event) -+{ -+} -+ -+static inline void kbase_timeline_pm_l2_transition_start(struct kbase_device *kbdev) -+{ -+} -+ -+static inline void kbase_timeline_pm_l2_transition_done(struct kbase_device *kbdev) -+{ -+} -+#endif /* CONFIG_MALI_TRACE_TIMELINE */ -+ -+#endif /* _KBASE_TRACE_TIMELINE_H */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h -new file mode 100755 -index 000000000..156a95a67 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_trace_timeline_defs.h -@@ -0,0 +1,140 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** -+ * ***** DO NOT INCLUDE DIRECTLY ***** -+ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ -+ -+/* -+ * Conventions on Event Names: -+ * -+ * - The prefix determines something about how the timeline should be -+ * displayed, and is split up into various parts, separated by underscores: -+ * - 'SW' and 'HW' as the first part will be used to determine whether a -+ * timeline is to do with Software or Hardware - effectively, separate -+ * 'channels' for Software and Hardware -+ * - 'START', 'STOP', 'ENTER', 'LEAVE' can be used in the second part, and -+ * signify related pairs of events - these are optional. -+ * - 'FLOW' indicates a generic event, which can use dependencies -+ * - This gives events such as: -+ * - 'SW_ENTER_FOO' -+ * - 'SW_LEAVE_FOO' -+ * - 'SW_FLOW_BAR_1' -+ * - 'SW_FLOW_BAR_2' -+ * - 'HW_START_BAZ' -+ * - 'HW_STOP_BAZ' -+ * - And an unadorned HW event: -+ * - 'HW_BAZ_FROZBOZ' -+ */ -+ -+/* -+ * Conventions on parameter names: -+ * - anything with 'instance' in the name will have a separate timeline based -+ * on that instances. -+ * - underscored-prefixed parameters will by hidden by default on timelines -+ * -+ * Hence: -+ * - Different job slots have their own 'instance', based on the instance value -+ * - Per-context info (e.g. atoms on a context) have their own 'instance' -+ * (i.e. each context should be on a different timeline) -+ * -+ * Note that globally-shared resources can be tagged with a tgid, but we don't -+ * want an instance per context: -+ * - There's no point having separate Job Slot timelines for each context, that -+ * would be confusing - there's only really 3 job slots! -+ * - There's no point having separate Shader-powered timelines for each -+ * context, that would be confusing - all shader cores (whether it be 4, 8, -+ * etc) are shared in the system. -+ */ -+ -+ /* -+ * CTX events -+ */ -+ /* Separate timelines for each context 'instance'*/ -+ KBASE_TIMELINE_TRACE_CODE(CTX_SET_NR_ATOMS_IN_FLIGHT, "CTX: Atoms in flight", "%d,%d", "_instance_tgid,_value_number_of_atoms"), -+ KBASE_TIMELINE_TRACE_CODE(CTX_FLOW_ATOM_READY, "CTX: Atoms Ready to Run", "%d,%d,%d", "_instance_tgid,_consumerof_atom_number,_producerof_atom_number_ready"), -+ -+ /* -+ * SW Events -+ */ -+ /* Separate timelines for each slot 'instance' */ -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_ACTIVE, "SW: GPU slot active", "%d,%d,%d", "_tgid,_instance_slot,_value_number_of_atoms"), -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_NEXT, "SW: GPU atom in NEXT", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_next"), -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_HEAD, "SW: GPU atom in HEAD", "%d,%d,%d", "_tgid,_instance_slot,_value_is_an_atom_in_head"), -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_SLOT_STOPPING, "SW: Try Soft-Stop on GPU slot", "%d,%d,%d", "_tgid,_instance_slot,_value_is_slot_stopping"), -+ /* Shader and overall power is shared - can't have separate instances of -+ * it, just tagging with the context */ -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_ACTIVE, "SW: GPU power active", "%d,%d", "_tgid,_value_is_power_active"), -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_TILER_ACTIVE, "SW: GPU tiler powered", "%d,%d", "_tgid,_value_number_of_tilers"), -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_SHADER_ACTIVE, "SW: GPU shaders powered", "%d,%d", "_tgid,_value_number_of_shaders"), -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powered", "%d,%d", "_tgid,_value_number_of_l2"), -+ -+ /* SW Power event messaging. _event_type is one from the kbase_pm_event enum */ -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_SEND_EVENT, "SW: PM Send Event", "%d,%d,%d", "_tgid,_event_type,_writerof_pm_event_id"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_HANDLE_EVENT, "SW: PM Handle Event", "%d,%d,%d", "_tgid,_event_type,_finalconsumerof_pm_event_id"), -+ /* SW L2 power events */ -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_POWERING, "SW: GPU L2 powering", "%d,%d", "_tgid,_writerof_l2_transitioning"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_GPU_POWER_L2_ACTIVE, "SW: GPU L2 powering done", "%d,%d", "_tgid,_finalconsumerof_l2_transitioning"), -+ -+ KBASE_TIMELINE_TRACE_CODE(SW_SET_CONTEXT_ACTIVE, "SW: Context Active", "%d,%d", "_tgid,_value_active"), -+ -+ /* -+ * BEGIN: Significant SW Functions that call kbase_pm_check_transitions_nolock() -+ */ -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweroff"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END, "SW: PM CheckTrans from kbase_pm_do_poweroff", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweroff"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_poweron"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END, "SW: PM CheckTrans from kbase_pm_do_poweron", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_poweron"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_writerof_pm_checktrans_gpu_interrupt"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END, "SW: PM CheckTrans from kbase_gpu_interrupt", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_gpu_interrupt"), -+ -+ /* -+ * Significant Indirect callers of kbase_pm_check_transitions_nolock() -+ */ -+ /* kbase_pm_request_cores */ -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_request_cores_shader_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_request_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_request_cores_shader_tiler"), -+ /* kbase_pm_release_cores */ -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_release_cores_shader_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END, "SW: PM CheckTrans from kbase_pm_release_cores(shader+tiler)", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_release_cores_shader_tiler"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_writerof_pm_checktrans_pm_do_shader_poweroff_callback"), -+ KBASE_TIMELINE_TRACE_CODE(SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END, "SW: PM CheckTrans from kbasep_pm_do_shader_poweroff_callback", "%d,%d", "_tgid,_finalconsumerof_pm_checktrans_pm_do_shader_poweroff_callback"), -+ /* -+ * END: SW Functions that call kbase_pm_check_transitions_nolock() -+ */ -+ -+ /* -+ * HW Events -+ */ -+ KBASE_TIMELINE_TRACE_CODE(HW_MMU_FAULT, -+"HW: MMU Fault", "%d,%d,%d", "_tgid,fault_type,fault_stage,asid"), -+ KBASE_TIMELINE_TRACE_CODE(HW_START_GPU_JOB_CHAIN_SW_APPROX, -+"HW: Job Chain start (SW approximated)", "%d,%d,%d", -+"_tgid,job_slot,_consumerof_atom_number_ready"), -+ KBASE_TIMELINE_TRACE_CODE(HW_STOP_GPU_JOB_CHAIN_SW_APPROX, -+"HW: Job Chain stop (SW approximated)", "%d,%d,%d", -+"_tgid,job_slot,_producerof_atom_number_completed") -diff --git a/drivers/gpu/arm/midgard/mali_kbase_uku.h b/drivers/gpu/arm/midgard/mali_kbase_uku.h -new file mode 100755 -index 000000000..c22a59324 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_uku.h -@@ -0,0 +1,545 @@ -+/* -+ * -+ * (C) COPYRIGHT 2008-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_UKU_H_ -+#define _KBASE_UKU_H_ -+ -+#include "mali_uk.h" -+#include "mali_base_kernel.h" -+ -+/* This file needs to support being included from kernel and userside (which use different defines) */ -+#if defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON -+#define SUPPORT_MALI_ERROR_INJECT -+#endif /* defined(CONFIG_MALI_ERROR_INJECT) || MALI_ERROR_INJECT_ON */ -+#if defined(CONFIG_MALI_NO_MALI) -+#define SUPPORT_MALI_NO_MALI -+#elif defined(MALI_NO_MALI) -+#if MALI_NO_MALI -+#define SUPPORT_MALI_NO_MALI -+#endif -+#endif -+ -+#if defined(SUPPORT_MALI_NO_MALI) || defined(SUPPORT_MALI_ERROR_INJECT) -+#include "backend/gpu/mali_kbase_model_dummy.h" -+#endif -+ -+#include "mali_kbase_gpuprops_types.h" -+ -+/* -+ * 10.1: -+ * - Do mmap in kernel for SAME_VA memory allocations rather then -+ * calling back into the kernel as a 2nd stage of the allocation request. -+ * -+ * 10.2: -+ * - Add KBASE_FUNC_MEM_JIT_INIT which allows clients to request a custom VA -+ * region for use with JIT (ignored on 32-bit platforms) -+ * -+ * 10.3: -+ * - base_jd_core_req typedef-ed to u32 (instead of to u16) -+ * - two flags added: BASE_JD_REQ_SKIP_CACHE_STAT / _END -+ * -+ * 10.4: -+ * - Removed KBASE_FUNC_EXT_BUFFER_LOCK used only in internal tests -+ * -+ * 10.5: -+ * - Reverted to performing mmap in user space so that tools like valgrind work. -+ * -+ * 10.6: -+ * - Add flags input variable to KBASE_FUNC_TLSTREAM_ACQUIRE -+ */ -+#define BASE_UK_VERSION_MAJOR 10 -+#define BASE_UK_VERSION_MINOR 6 -+ -+#define LINUX_UK_BASE_MAGIC 0x80 -+ -+struct kbase_uk_mem_alloc { -+ union uk_header header; -+ /* IN */ -+ u64 va_pages; -+ u64 commit_pages; -+ u64 extent; -+ /* IN/OUT */ -+ u64 flags; -+ /* OUT */ -+ u64 gpu_va; -+ u16 va_alignment; -+ u8 padding[6]; -+}; -+ -+struct kbase_uk_mem_free { -+ union uk_header header; -+ /* IN */ -+ u64 gpu_addr; -+ /* OUT */ -+}; -+ -+struct kbase_uk_mem_alias { -+ union uk_header header; -+ /* IN/OUT */ -+ u64 flags; -+ /* IN */ -+ u64 stride; -+ u64 nents; -+ union kbase_pointer ai; -+ /* OUT */ -+ u64 gpu_va; -+ u64 va_pages; -+}; -+ -+struct kbase_uk_mem_import { -+ union uk_header header; -+ /* IN */ -+ union kbase_pointer phandle; -+ u32 type; -+ u32 padding; -+ /* IN/OUT */ -+ u64 flags; -+ /* OUT */ -+ u64 gpu_va; -+ u64 va_pages; -+}; -+ -+struct kbase_uk_mem_flags_change { -+ union uk_header header; -+ /* IN */ -+ u64 gpu_va; -+ u64 flags; -+ u64 mask; -+}; -+ -+struct kbase_uk_job_submit { -+ union uk_header header; -+ /* IN */ -+ union kbase_pointer addr; -+ u32 nr_atoms; -+ u32 stride; /* bytes between atoms, i.e. sizeof(base_jd_atom_v2) */ -+ /* OUT */ -+}; -+ -+struct kbase_uk_post_term { -+ union uk_header header; -+}; -+ -+struct kbase_uk_sync_now { -+ union uk_header header; -+ -+ /* IN */ -+ struct base_syncset sset; -+ -+ /* OUT */ -+}; -+ -+struct kbase_uk_hwcnt_setup { -+ union uk_header header; -+ -+ /* IN */ -+ u64 dump_buffer; -+ u32 jm_bm; -+ u32 shader_bm; -+ u32 tiler_bm; -+ u32 unused_1; /* keep for backwards compatibility */ -+ u32 mmu_l2_bm; -+ u32 padding; -+ /* OUT */ -+}; -+ -+/** -+ * struct kbase_uk_hwcnt_reader_setup - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * @buffer_count: requested number of dumping buffers -+ * @jm_bm: counters selection bitmask (JM) -+ * @shader_bm: counters selection bitmask (Shader) -+ * @tiler_bm: counters selection bitmask (Tiler) -+ * @mmu_l2_bm: counters selection bitmask (MMU_L2) -+ * @fd: dumping notification file descriptor -+ * -+ * This structure sets up HWC dumper/reader for this context. -+ * Multiple instances can be created for single context. -+ */ -+struct kbase_uk_hwcnt_reader_setup { -+ union uk_header header; -+ -+ /* IN */ -+ u32 buffer_count; -+ u32 jm_bm; -+ u32 shader_bm; -+ u32 tiler_bm; -+ u32 mmu_l2_bm; -+ -+ /* OUT */ -+ s32 fd; -+}; -+ -+struct kbase_uk_hwcnt_dump { -+ union uk_header header; -+}; -+ -+struct kbase_uk_hwcnt_clear { -+ union uk_header header; -+}; -+ -+struct kbase_uk_fence_validate { -+ union uk_header header; -+ /* IN */ -+ s32 fd; -+ u32 padding; -+ /* OUT */ -+}; -+ -+struct kbase_uk_stream_create { -+ union uk_header header; -+ /* IN */ -+ char name[32]; -+ /* OUT */ -+ s32 fd; -+ u32 padding; -+}; -+ -+struct kbase_uk_gpuprops { -+ union uk_header header; -+ -+ /* IN */ -+ struct mali_base_gpu_props props; -+ /* OUT */ -+}; -+ -+struct kbase_uk_mem_query { -+ union uk_header header; -+ /* IN */ -+ u64 gpu_addr; -+#define KBASE_MEM_QUERY_COMMIT_SIZE 1 -+#define KBASE_MEM_QUERY_VA_SIZE 2 -+#define KBASE_MEM_QUERY_FLAGS 3 -+ u64 query; -+ /* OUT */ -+ u64 value; -+}; -+ -+struct kbase_uk_mem_commit { -+ union uk_header header; -+ /* IN */ -+ u64 gpu_addr; -+ u64 pages; -+ /* OUT */ -+ u32 result_subcode; -+ u32 padding; -+}; -+ -+struct kbase_uk_find_cpu_offset { -+ union uk_header header; -+ /* IN */ -+ u64 gpu_addr; -+ u64 cpu_addr; -+ u64 size; -+ /* OUT */ -+ u64 offset; -+}; -+ -+#define KBASE_GET_VERSION_BUFFER_SIZE 64 -+struct kbase_uk_get_ddk_version { -+ union uk_header header; -+ /* OUT */ -+ char version_buffer[KBASE_GET_VERSION_BUFFER_SIZE]; -+ u32 version_string_size; -+ u32 padding; -+ u32 rk_version; -+}; -+ -+struct kbase_uk_disjoint_query { -+ union uk_header header; -+ /* OUT */ -+ u32 counter; -+ u32 padding; -+}; -+ -+struct kbase_uk_set_flags { -+ union uk_header header; -+ /* IN */ -+ u32 create_flags; -+ u32 padding; -+}; -+ -+#if MALI_UNIT_TEST -+#define TEST_ADDR_COUNT 4 -+#define KBASE_TEST_BUFFER_SIZE 128 -+struct kbase_exported_test_data { -+ u64 test_addr[TEST_ADDR_COUNT]; /**< memory address */ -+ u32 test_addr_pages[TEST_ADDR_COUNT]; /**< memory size in pages */ -+ union kbase_pointer kctx; /**< base context created by process */ -+ union kbase_pointer mm; /**< pointer to process address space */ -+ u8 buffer1[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ -+ u8 buffer2[KBASE_TEST_BUFFER_SIZE]; /**< unit test defined parameter */ -+}; -+ -+struct kbase_uk_set_test_data { -+ union uk_header header; -+ /* IN */ -+ struct kbase_exported_test_data test_data; -+}; -+ -+#endif /* MALI_UNIT_TEST */ -+ -+#ifdef SUPPORT_MALI_ERROR_INJECT -+struct kbase_uk_error_params { -+ union uk_header header; -+ /* IN */ -+ struct kbase_error_params params; -+}; -+#endif /* SUPPORT_MALI_ERROR_INJECT */ -+ -+#ifdef SUPPORT_MALI_NO_MALI -+struct kbase_uk_model_control_params { -+ union uk_header header; -+ /* IN */ -+ struct kbase_model_control_params params; -+}; -+#endif /* SUPPORT_MALI_NO_MALI */ -+ -+#ifdef BASE_LEGACY_UK8_SUPPORT -+struct kbase_uk_keep_gpu_powered { -+ union uk_header header; -+ u32 enabled; -+ u32 padding; -+}; -+#endif /* BASE_LEGACY_UK8_SUPPORT */ -+ -+struct kbase_uk_profiling_controls { -+ union uk_header header; -+ u32 profiling_controls[FBDUMP_CONTROL_MAX]; -+}; -+ -+struct kbase_uk_debugfs_mem_profile_add { -+ union uk_header header; -+ u32 len; -+ u32 padding; -+ union kbase_pointer buf; -+}; -+ -+struct kbase_uk_context_id { -+ union uk_header header; -+ /* OUT */ -+ int id; -+}; -+ -+/** -+ * struct kbase_uk_tlstream_acquire - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * @flags: timeline stream flags -+ * @fd: timeline stream file descriptor -+ * -+ * This structure is used when performing a call to acquire kernel side timeline -+ * stream file descriptor. -+ */ -+struct kbase_uk_tlstream_acquire { -+ union uk_header header; -+ /* IN */ -+ u32 flags; -+ /* OUT */ -+ s32 fd; -+}; -+ -+/** -+ * struct kbase_uk_tlstream_acquire_v10_4 - User/Kernel space data exchange -+ * structure -+ * @header: UK structure header -+ * @fd: timeline stream file descriptor -+ * -+ * This structure is used when performing a call to acquire kernel side timeline -+ * stream file descriptor. -+ */ -+struct kbase_uk_tlstream_acquire_v10_4 { -+ union uk_header header; -+ /* IN */ -+ /* OUT */ -+ s32 fd; -+}; -+ -+/** -+ * struct kbase_uk_tlstream_flush - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * -+ * This structure is used when performing a call to flush kernel side -+ * timeline streams. -+ */ -+struct kbase_uk_tlstream_flush { -+ union uk_header header; -+ /* IN */ -+ /* OUT */ -+}; -+ -+#if MALI_UNIT_TEST -+/** -+ * struct kbase_uk_tlstream_test - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * @tpw_count: number of trace point writers in each context -+ * @msg_delay: time delay between tracepoints from one writer in milliseconds -+ * @msg_count: number of trace points written by one writer -+ * @aux_msg: if non-zero aux messages will be included -+ * -+ * This structure is used when performing a call to start timeline stream test -+ * embedded in kernel. -+ */ -+struct kbase_uk_tlstream_test { -+ union uk_header header; -+ /* IN */ -+ u32 tpw_count; -+ u32 msg_delay; -+ u32 msg_count; -+ u32 aux_msg; -+ /* OUT */ -+}; -+ -+/** -+ * struct kbase_uk_tlstream_stats - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * @bytes_collected: number of bytes read by user -+ * @bytes_generated: number of bytes generated by tracepoints -+ * -+ * This structure is used when performing a call to obtain timeline stream -+ * statistics. -+ */ -+struct kbase_uk_tlstream_stats { -+ union uk_header header; /**< UK structure header. */ -+ /* IN */ -+ /* OUT */ -+ u32 bytes_collected; -+ u32 bytes_generated; -+}; -+#endif /* MALI_UNIT_TEST */ -+ -+/** -+ * struct struct kbase_uk_prfcnt_value for the KBASE_FUNC_SET_PRFCNT_VALUES ioctl -+ * @header: UK structure header -+ * @data: Counter samples for the dummy model -+ * @size:............Size of the counter sample data -+ */ -+struct kbase_uk_prfcnt_values { -+ union uk_header header; -+ /* IN */ -+ u32 *data; -+ u32 size; -+}; -+ -+/** -+ * struct kbase_uk_soft_event_update - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * @evt: the GPU address containing the event -+ * @new_status: the new event status, must be either BASE_JD_SOFT_EVENT_SET or -+ * BASE_JD_SOFT_EVENT_RESET -+ * @flags: reserved for future uses, must be set to 0 -+ * -+ * This structure is used to update the status of a software event. If the -+ * event's status is set to BASE_JD_SOFT_EVENT_SET, any job currently waiting -+ * on this event will complete. -+ */ -+struct kbase_uk_soft_event_update { -+ union uk_header header; -+ /* IN */ -+ u64 evt; -+ u32 new_status; -+ u32 flags; -+}; -+ -+/** -+ * struct kbase_uk_mem_jit_init - User/Kernel space data exchange structure -+ * @header: UK structure header -+ * @va_pages: Number of virtual pages required for JIT -+ * -+ * This structure is used when requesting initialization of JIT. -+ */ -+struct kbase_uk_mem_jit_init { -+ union uk_header header; -+ /* IN */ -+ u64 va_pages; -+}; -+ -+enum kbase_uk_function_id { -+ KBASE_FUNC_MEM_ALLOC = (UK_FUNC_ID + 0), -+ KBASE_FUNC_MEM_IMPORT = (UK_FUNC_ID + 1), -+ KBASE_FUNC_MEM_COMMIT = (UK_FUNC_ID + 2), -+ KBASE_FUNC_MEM_QUERY = (UK_FUNC_ID + 3), -+ KBASE_FUNC_MEM_FREE = (UK_FUNC_ID + 4), -+ KBASE_FUNC_MEM_FLAGS_CHANGE = (UK_FUNC_ID + 5), -+ KBASE_FUNC_MEM_ALIAS = (UK_FUNC_ID + 6), -+ -+#ifdef BASE_LEGACY_UK6_SUPPORT -+ KBASE_FUNC_JOB_SUBMIT_UK6 = (UK_FUNC_ID + 7), -+#endif /* BASE_LEGACY_UK6_SUPPORT */ -+ -+ KBASE_FUNC_SYNC = (UK_FUNC_ID + 8), -+ -+ KBASE_FUNC_POST_TERM = (UK_FUNC_ID + 9), -+ -+ KBASE_FUNC_HWCNT_SETUP = (UK_FUNC_ID + 10), -+ KBASE_FUNC_HWCNT_DUMP = (UK_FUNC_ID + 11), -+ KBASE_FUNC_HWCNT_CLEAR = (UK_FUNC_ID + 12), -+ -+ KBASE_FUNC_GPU_PROPS_REG_DUMP = (UK_FUNC_ID + 14), -+ -+ KBASE_FUNC_FIND_CPU_OFFSET = (UK_FUNC_ID + 15), -+ -+ KBASE_FUNC_GET_VERSION = (UK_FUNC_ID + 16), -+ KBASE_FUNC_SET_FLAGS = (UK_FUNC_ID + 18), -+ -+ KBASE_FUNC_SET_TEST_DATA = (UK_FUNC_ID + 19), -+ KBASE_FUNC_INJECT_ERROR = (UK_FUNC_ID + 20), -+ KBASE_FUNC_MODEL_CONTROL = (UK_FUNC_ID + 21), -+ -+#ifdef BASE_LEGACY_UK8_SUPPORT -+ KBASE_FUNC_KEEP_GPU_POWERED = (UK_FUNC_ID + 22), -+#endif /* BASE_LEGACY_UK8_SUPPORT */ -+ -+ KBASE_FUNC_FENCE_VALIDATE = (UK_FUNC_ID + 23), -+ KBASE_FUNC_STREAM_CREATE = (UK_FUNC_ID + 24), -+ KBASE_FUNC_GET_PROFILING_CONTROLS = (UK_FUNC_ID + 25), -+ KBASE_FUNC_SET_PROFILING_CONTROLS = (UK_FUNC_ID + 26), -+ /* to be used only for testing -+ * purposes, otherwise these controls -+ * are set through gator API */ -+ -+ KBASE_FUNC_DEBUGFS_MEM_PROFILE_ADD = (UK_FUNC_ID + 27), -+ KBASE_FUNC_JOB_SUBMIT = (UK_FUNC_ID + 28), -+ KBASE_FUNC_DISJOINT_QUERY = (UK_FUNC_ID + 29), -+ -+ KBASE_FUNC_GET_CONTEXT_ID = (UK_FUNC_ID + 31), -+ -+ KBASE_FUNC_TLSTREAM_ACQUIRE_V10_4 = (UK_FUNC_ID + 32), -+#if MALI_UNIT_TEST -+ KBASE_FUNC_TLSTREAM_TEST = (UK_FUNC_ID + 33), -+ KBASE_FUNC_TLSTREAM_STATS = (UK_FUNC_ID + 34), -+#endif /* MALI_UNIT_TEST */ -+ KBASE_FUNC_TLSTREAM_FLUSH = (UK_FUNC_ID + 35), -+ -+ KBASE_FUNC_HWCNT_READER_SETUP = (UK_FUNC_ID + 36), -+ -+#ifdef SUPPORT_MALI_NO_MALI -+ KBASE_FUNC_SET_PRFCNT_VALUES = (UK_FUNC_ID + 37), -+#endif -+ -+ KBASE_FUNC_SOFT_EVENT_UPDATE = (UK_FUNC_ID + 38), -+ -+ KBASE_FUNC_MEM_JIT_INIT = (UK_FUNC_ID + 39), -+ -+ KBASE_FUNC_TLSTREAM_ACQUIRE = (UK_FUNC_ID + 40), -+ -+ KBASE_FUNC_MAX -+}; -+ -+#endif /* _KBASE_UKU_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_utility.c b/drivers/gpu/arm/midgard/mali_kbase_utility.c -new file mode 100755 -index 000000000..be474ff87 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_utility.c -@@ -0,0 +1,33 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+ -+bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry) -+{ -+ struct list_head *pos = base->next; -+ -+ while (pos != base) { -+ if (pos == entry) -+ return true; -+ -+ pos = pos->next; -+ } -+ return false; -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_utility.h b/drivers/gpu/arm/midgard/mali_kbase_utility.h -new file mode 100755 -index 000000000..fd7252dab ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_utility.h -@@ -0,0 +1,37 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_UTILITY_H -+#define _KBASE_UTILITY_H -+ -+#ifndef _KBASE_H_ -+#error "Don't include this file directly, use mali_kbase.h instead" -+#endif -+ -+/** Test whether the given list entry is a member of the given list. -+ * -+ * @param base The head of the list to be tested -+ * @param entry The list entry to be tested -+ * -+ * @return true if entry is a member of base -+ * false otherwise -+ */ -+bool kbasep_list_member_of(const struct list_head *base, struct list_head *entry); -+ -+#endif /* _KBASE_UTILITY_H */ -diff --git a/drivers/gpu/arm/midgard/mali_kbase_vinstr.c b/drivers/gpu/arm/midgard/mali_kbase_vinstr.c -new file mode 100755 -index 000000000..8395568d0 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_vinstr.c -@@ -0,0 +1,2070 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+/*****************************************************************************/ -+ -+/* Hwcnt reader API version */ -+#define HWCNT_READER_API 1 -+ -+/* The number of nanoseconds in a second. */ -+#define NSECS_IN_SEC 1000000000ull /* ns */ -+ -+/* The time resolution of dumping service. */ -+#define DUMPING_RESOLUTION 500000ull /* ns */ -+ -+/* The maximal supported number of dumping buffers. */ -+#define MAX_BUFFER_COUNT 32 -+ -+/* Size and number of hw counters blocks. */ -+#define NR_CNT_BLOCKS_PER_GROUP 8 -+#define NR_CNT_PER_BLOCK 64 -+#define NR_BYTES_PER_CNT 4 -+#define NR_BYTES_PER_HDR 16 -+#define PRFCNT_EN_MASK_OFFSET 0x8 -+ -+/*****************************************************************************/ -+ -+enum { -+ SHADER_HWCNT_BM, -+ TILER_HWCNT_BM, -+ MMU_L2_HWCNT_BM, -+ JM_HWCNT_BM -+}; -+ -+enum vinstr_state { -+ VINSTR_IDLE, -+ VINSTR_DUMPING, -+ VINSTR_SUSPENDING, -+ VINSTR_SUSPENDED, -+ VINSTR_RESUMING -+}; -+ -+/** -+ * struct kbase_vinstr_context - vinstr context per device -+ * @lock: protects the entire vinstr context -+ * @kbdev: pointer to kbase device -+ * @kctx: pointer to kbase context -+ * @vmap: vinstr vmap for mapping hwcnt dump buffer -+ * @gpu_va: GPU hwcnt dump buffer address -+ * @cpu_va: the CPU side mapping of the hwcnt dump buffer -+ * @dump_size: size of the dump buffer in bytes -+ * @bitmap: current set of counters monitored, not always in sync -+ * with hardware -+ * @reprogram: when true, reprogram hwcnt block with the new set of -+ * counters -+ * @state: vinstr state -+ * @state_lock: protects information about vinstr state -+ * @suspend_waitq: notification queue to trigger state re-validation -+ * @suspend_cnt: reference counter of vinstr's suspend state -+ * @suspend_work: worker to execute on entering suspended state -+ * @resume_work: worker to execute on leaving suspended state -+ * @nclients: number of attached clients, pending or otherwise -+ * @waiting_clients: head of list of clients being periodically sampled -+ * @idle_clients: head of list of clients being idle -+ * @suspended_clients: head of list of clients being suspended -+ * @thread: periodic sampling thread -+ * @waitq: notification queue of sampling thread -+ * @request_pending: request for action for sampling thread -+ */ -+struct kbase_vinstr_context { -+ struct mutex lock; -+ struct kbase_device *kbdev; -+ struct kbase_context *kctx; -+ -+ struct kbase_vmap_struct vmap; -+ u64 gpu_va; -+ void *cpu_va; -+ size_t dump_size; -+ u32 bitmap[4]; -+ bool reprogram; -+ -+ enum vinstr_state state; -+ struct spinlock state_lock; -+ wait_queue_head_t suspend_waitq; -+ unsigned int suspend_cnt; -+ struct work_struct suspend_work; -+ struct work_struct resume_work; -+ -+ u32 nclients; -+ struct list_head waiting_clients; -+ struct list_head idle_clients; -+ struct list_head suspended_clients; -+ -+ struct task_struct *thread; -+ wait_queue_head_t waitq; -+ atomic_t request_pending; -+}; -+ -+/** -+ * struct kbase_vinstr_client - a vinstr client attached to a vinstr context -+ * @vinstr_ctx: vinstr context client is attached to -+ * @list: node used to attach this client to list in vinstr context -+ * @buffer_count: number of buffers this client is using -+ * @event_mask: events this client reacts to -+ * @dump_size: size of one dump buffer in bytes -+ * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters -+ * @legacy_buffer: userspace hwcnt dump buffer (legacy interface) -+ * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface) -+ * @accum_buffer: temporary accumulation buffer for preserving counters -+ * @dump_time: next time this clients shall request hwcnt dump -+ * @dump_interval: interval between periodic hwcnt dumps -+ * @dump_buffers: kernel hwcnt dump buffers allocated by this client -+ * @dump_buffers_meta: metadata of dump buffers -+ * @meta_idx: index of metadata being accessed by userspace -+ * @read_idx: index of buffer read by userspace -+ * @write_idx: index of buffer being written by dumping service -+ * @waitq: client's notification queue -+ * @pending: when true, client has attached but hwcnt not yet updated -+ */ -+struct kbase_vinstr_client { -+ struct kbase_vinstr_context *vinstr_ctx; -+ struct list_head list; -+ unsigned int buffer_count; -+ u32 event_mask; -+ size_t dump_size; -+ u32 bitmap[4]; -+ void __user *legacy_buffer; -+ void *kernel_buffer; -+ void *accum_buffer; -+ u64 dump_time; -+ u32 dump_interval; -+ char *dump_buffers; -+ struct kbase_hwcnt_reader_metadata *dump_buffers_meta; -+ atomic_t meta_idx; -+ atomic_t read_idx; -+ atomic_t write_idx; -+ wait_queue_head_t waitq; -+ bool pending; -+}; -+ -+/** -+ * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer -+ * @hrtimer: high resolution timer -+ * @vinstr_ctx: vinstr context -+ */ -+struct kbasep_vinstr_wake_up_timer { -+ struct hrtimer hrtimer; -+ struct kbase_vinstr_context *vinstr_ctx; -+}; -+ -+/*****************************************************************************/ -+ -+static int kbasep_vinstr_service_task(void *data); -+ -+static unsigned int kbasep_vinstr_hwcnt_reader_poll( -+ struct file *filp, -+ poll_table *wait); -+static long kbasep_vinstr_hwcnt_reader_ioctl( -+ struct file *filp, -+ unsigned int cmd, -+ unsigned long arg); -+static int kbasep_vinstr_hwcnt_reader_mmap( -+ struct file *filp, -+ struct vm_area_struct *vma); -+static int kbasep_vinstr_hwcnt_reader_release( -+ struct inode *inode, -+ struct file *filp); -+ -+/* The timeline stream file operations structure. */ -+static const struct file_operations vinstr_client_fops = { -+ .poll = kbasep_vinstr_hwcnt_reader_poll, -+ .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, -+ .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl, -+ .mmap = kbasep_vinstr_hwcnt_reader_mmap, -+ .release = kbasep_vinstr_hwcnt_reader_release, -+}; -+ -+/*****************************************************************************/ -+ -+static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_context *kctx = vinstr_ctx->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_uk_hwcnt_setup setup; -+ int err; -+ -+ setup.dump_buffer = vinstr_ctx->gpu_va; -+ setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM]; -+ setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM]; -+ setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM]; -+ setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM]; -+ -+ /* Mark the context as active so the GPU is kept turned on */ -+ /* A suspend won't happen here, because we're in a syscall from a -+ * userspace thread. */ -+ kbase_pm_context_active(kbdev); -+ -+ /* Schedule the context in */ -+ kbasep_js_schedule_privileged_ctx(kbdev, kctx); -+ err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup); -+ if (err) { -+ /* Release the context. This had its own Power Manager Active -+ * reference */ -+ kbasep_js_release_privileged_ctx(kbdev, kctx); -+ -+ /* Also release our Power Manager Active reference */ -+ kbase_pm_context_idle(kbdev); -+ } -+ -+ return err; -+} -+ -+static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_context *kctx = vinstr_ctx->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ int err; -+ -+ err = kbase_instr_hwcnt_disable_internal(kctx); -+ if (err) { -+ dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)", -+ kctx); -+ return; -+ } -+ -+ /* Release the context. This had its own Power Manager Active reference. */ -+ kbasep_js_release_privileged_ctx(kbdev, kctx); -+ -+ /* Also release our Power Manager Active reference. */ -+ kbase_pm_context_idle(kbdev); -+ -+ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx); -+} -+ -+static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ disable_hwcnt(vinstr_ctx); -+ return enable_hwcnt(vinstr_ctx); -+} -+ -+static void hwcnt_bitmap_set(u32 dst[4], u32 src[4]) -+{ -+ dst[JM_HWCNT_BM] = src[JM_HWCNT_BM]; -+ dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM]; -+ dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM]; -+ dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM]; -+} -+ -+static void hwcnt_bitmap_union(u32 dst[4], u32 src[4]) -+{ -+ dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM]; -+ dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM]; -+ dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM]; -+ dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM]; -+} -+ -+size_t kbase_vinstr_dump_size(struct kbase_device *kbdev) -+{ -+ size_t dump_size; -+ -+#ifndef CONFIG_MALI_NO_MALI -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) { -+ u32 nr_cg; -+ -+ nr_cg = kbdev->gpu_props.num_core_groups; -+ dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP * -+ NR_CNT_PER_BLOCK * -+ NR_BYTES_PER_CNT; -+ } else -+#endif /* CONFIG_MALI_NO_MALI */ -+ { -+ /* assume v5 for now */ -+ base_gpu_props *props = &kbdev->gpu_props.props; -+ u32 nr_l2 = props->l2_props.num_l2_slices; -+ u64 core_mask = props->coherency_info.group[0].core_mask; -+ u32 nr_blocks = fls64(core_mask); -+ -+ /* JM and tiler counter blocks are always present */ -+ dump_size = (2 + nr_l2 + nr_blocks) * -+ NR_CNT_PER_BLOCK * -+ NR_BYTES_PER_CNT; -+ } -+ return dump_size; -+} -+KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size); -+ -+static size_t kbasep_vinstr_dump_size_ctx( -+ struct kbase_vinstr_context *vinstr_ctx) -+{ -+ return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev); -+} -+ -+static int kbasep_vinstr_map_kernel_dump_buffer( -+ struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_va_region *reg; -+ struct kbase_context *kctx = vinstr_ctx->kctx; -+ u64 flags, nr_pages; -+ -+ flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR; -+ vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); -+ nr_pages = PFN_UP(vinstr_ctx->dump_size); -+ -+ reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, -+ &vinstr_ctx->gpu_va); -+ if (!reg) -+ return -ENOMEM; -+ -+ vinstr_ctx->cpu_va = kbase_vmap( -+ kctx, -+ vinstr_ctx->gpu_va, -+ vinstr_ctx->dump_size, -+ &vinstr_ctx->vmap); -+ if (!vinstr_ctx->cpu_va) { -+ kbase_mem_free(kctx, vinstr_ctx->gpu_va); -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+static void kbasep_vinstr_unmap_kernel_dump_buffer( -+ struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_context *kctx = vinstr_ctx->kctx; -+ -+ kbase_vunmap(kctx, &vinstr_ctx->vmap); -+ kbase_mem_free(kctx, vinstr_ctx->gpu_va); -+} -+ -+/** -+ * kbasep_vinstr_create_kctx - create kernel context for vinstr -+ * @vinstr_ctx: vinstr context -+ * Return: zero on success -+ */ -+static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_device *kbdev = vinstr_ctx->kbdev; -+ struct kbasep_kctx_list_element *element; -+ unsigned long flags; -+ bool enable_backend = false; -+ int err; -+ -+ vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true); -+ if (!vinstr_ctx->kctx) -+ return -ENOMEM; -+ -+ /* Map the master kernel dump buffer. The HW dumps the counters -+ * into this memory region. */ -+ err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx); -+ if (err) { -+ kbase_destroy_context(vinstr_ctx->kctx); -+ vinstr_ctx->kctx = NULL; -+ return err; -+ } -+ -+ /* Add kernel context to list of contexts associated with device. */ -+ element = kzalloc(sizeof(*element), GFP_KERNEL); -+ if (element) { -+ element->kctx = vinstr_ctx->kctx; -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_add(&element->link, &kbdev->kctx_list); -+ -+ /* Inform timeline client about new context. -+ * Do this while holding the lock to avoid tracepoint -+ * being created in both body and summary stream. */ -+ KBASE_TLSTREAM_TL_NEW_CTX( -+ vinstr_ctx->kctx, -+ (u32)(vinstr_ctx->kctx->id), -+ (u32)(vinstr_ctx->kctx->tgid)); -+ -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } else { -+ /* Don't treat this as a fail - just warn about it. */ -+ dev_warn(kbdev->dev, -+ "couldn't add kctx to kctx_list\n"); -+ } -+ -+ /* Don't enable hardware counters if vinstr is suspended. -+ * Note that vinstr resume code is run under vinstr context lock, -+ * lower layer will be enabled as needed on resume. */ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ if (VINSTR_IDLE == vinstr_ctx->state) -+ enable_backend = true; -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ if (enable_backend) -+ err = enable_hwcnt(vinstr_ctx); -+ -+ if (err) { -+ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); -+ kbase_destroy_context(vinstr_ctx->kctx); -+ if (element) { -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_del(&element->link); -+ kfree(element); -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } -+ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); -+ vinstr_ctx->kctx = NULL; -+ return err; -+ } -+ -+ vinstr_ctx->thread = kthread_run( -+ kbasep_vinstr_service_task, -+ vinstr_ctx, -+ "mali_vinstr_service"); -+ if (!vinstr_ctx->thread) { -+ disable_hwcnt(vinstr_ctx); -+ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); -+ kbase_destroy_context(vinstr_ctx->kctx); -+ if (element) { -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_del(&element->link); -+ kfree(element); -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } -+ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); -+ vinstr_ctx->kctx = NULL; -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context -+ * @vinstr_ctx: vinstr context -+ */ -+static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_device *kbdev = vinstr_ctx->kbdev; -+ struct kbasep_kctx_list_element *element; -+ struct kbasep_kctx_list_element *tmp; -+ bool found = false; -+ -+ /* Release hw counters dumping resources. */ -+ vinstr_ctx->thread = NULL; -+ disable_hwcnt(vinstr_ctx); -+ kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx); -+ kbase_destroy_context(vinstr_ctx->kctx); -+ -+ /* Remove kernel context from the device's contexts list. */ -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) { -+ if (element->kctx == vinstr_ctx->kctx) { -+ list_del(&element->link); -+ kfree(element); -+ found = true; -+ } -+ } -+ mutex_unlock(&kbdev->kctx_list_lock); -+ -+ if (!found) -+ dev_warn(kbdev->dev, "kctx not in kctx_list\n"); -+ -+ /* Inform timeline client about context destruction. */ -+ KBASE_TLSTREAM_TL_DEL_CTX(vinstr_ctx->kctx); -+ -+ vinstr_ctx->kctx = NULL; -+} -+ -+/** -+ * kbasep_vinstr_attach_client - Attach a client to the vinstr core -+ * @vinstr_ctx: vinstr context -+ * @buffer_count: requested number of dump buffers -+ * @bitmap: bitmaps describing which counters should be enabled -+ * @argp: pointer where notification descriptor shall be stored -+ * @kernel_buffer: pointer to kernel side buffer -+ * -+ * Return: vinstr opaque client handle or NULL on failure -+ */ -+static struct kbase_vinstr_client *kbasep_vinstr_attach_client( -+ struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count, -+ u32 bitmap[4], void *argp, void *kernel_buffer) -+{ -+ struct task_struct *thread = NULL; -+ struct kbase_vinstr_client *cli; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ if (buffer_count > MAX_BUFFER_COUNT -+ || (buffer_count & (buffer_count - 1))) -+ return NULL; -+ -+ cli = kzalloc(sizeof(*cli), GFP_KERNEL); -+ if (!cli) -+ return NULL; -+ -+ cli->vinstr_ctx = vinstr_ctx; -+ cli->buffer_count = buffer_count; -+ cli->event_mask = -+ (1 << BASE_HWCNT_READER_EVENT_MANUAL) | -+ (1 << BASE_HWCNT_READER_EVENT_PERIODIC); -+ cli->pending = true; -+ -+ hwcnt_bitmap_set(cli->bitmap, bitmap); -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap); -+ vinstr_ctx->reprogram = true; -+ -+ /* If this is the first client, create the vinstr kbase -+ * context. This context is permanently resident until the -+ * last client exits. */ -+ if (!vinstr_ctx->nclients) { -+ hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap); -+ if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0) -+ goto error; -+ -+ vinstr_ctx->reprogram = false; -+ cli->pending = false; -+ } -+ -+ /* The GPU resets the counter block every time there is a request -+ * to dump it. We need a per client kernel buffer for accumulating -+ * the counters. */ -+ cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx); -+ cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL); -+ if (!cli->accum_buffer) -+ goto error; -+ -+ /* Prepare buffers. */ -+ if (cli->buffer_count) { -+ int *fd = (int *)argp; -+ size_t tmp; -+ -+ /* Allocate area for buffers metadata storage. */ -+ tmp = sizeof(struct kbase_hwcnt_reader_metadata) * -+ cli->buffer_count; -+ cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL); -+ if (!cli->dump_buffers_meta) -+ goto error; -+ -+ /* Allocate required number of dumping buffers. */ -+ cli->dump_buffers = (char *)__get_free_pages( -+ GFP_KERNEL | __GFP_ZERO, -+ get_order(cli->dump_size * cli->buffer_count)); -+ if (!cli->dump_buffers) -+ goto error; -+ -+ /* Create descriptor for user-kernel data exchange. */ -+ *fd = anon_inode_getfd( -+ "[mali_vinstr_desc]", -+ &vinstr_client_fops, -+ cli, -+ O_RDONLY | O_CLOEXEC); -+ if (0 > *fd) -+ goto error; -+ } else if (kernel_buffer) { -+ cli->kernel_buffer = kernel_buffer; -+ } else { -+ cli->legacy_buffer = (void __user *)argp; -+ } -+ -+ atomic_set(&cli->read_idx, 0); -+ atomic_set(&cli->meta_idx, 0); -+ atomic_set(&cli->write_idx, 0); -+ init_waitqueue_head(&cli->waitq); -+ -+ vinstr_ctx->nclients++; -+ list_add(&cli->list, &vinstr_ctx->idle_clients); -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ return cli; -+ -+error: -+ kfree(cli->dump_buffers_meta); -+ if (cli->dump_buffers) -+ free_pages( -+ (unsigned long)cli->dump_buffers, -+ get_order(cli->dump_size * cli->buffer_count)); -+ kfree(cli->accum_buffer); -+ if (!vinstr_ctx->nclients && vinstr_ctx->kctx) { -+ thread = vinstr_ctx->thread; -+ kbasep_vinstr_destroy_kctx(vinstr_ctx); -+ } -+ kfree(cli); -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ /* Thread must be stopped after lock is released. */ -+ if (thread) -+ kthread_stop(thread); -+ -+ return NULL; -+} -+ -+void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli) -+{ -+ struct kbase_vinstr_context *vinstr_ctx; -+ struct kbase_vinstr_client *iter, *tmp; -+ struct task_struct *thread = NULL; -+ u32 zerobitmap[4] = { 0 }; -+ int cli_found = 0; -+ -+ KBASE_DEBUG_ASSERT(cli); -+ vinstr_ctx = cli->vinstr_ctx; -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) { -+ if (iter == cli) { -+ vinstr_ctx->reprogram = true; -+ cli_found = 1; -+ list_del(&iter->list); -+ break; -+ } -+ } -+ if (!cli_found) { -+ list_for_each_entry_safe( -+ iter, tmp, &vinstr_ctx->waiting_clients, list) { -+ if (iter == cli) { -+ vinstr_ctx->reprogram = true; -+ cli_found = 1; -+ list_del(&iter->list); -+ break; -+ } -+ } -+ } -+ KBASE_DEBUG_ASSERT(cli_found); -+ -+ kfree(cli->dump_buffers_meta); -+ free_pages( -+ (unsigned long)cli->dump_buffers, -+ get_order(cli->dump_size * cli->buffer_count)); -+ kfree(cli->accum_buffer); -+ kfree(cli); -+ -+ vinstr_ctx->nclients--; -+ if (!vinstr_ctx->nclients) { -+ thread = vinstr_ctx->thread; -+ kbasep_vinstr_destroy_kctx(vinstr_ctx); -+ } -+ -+ /* Rebuild context bitmap now that the client has detached */ -+ hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap); -+ list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) -+ hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); -+ list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) -+ hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap); -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ /* Thread must be stopped after lock is released. */ -+ if (thread) -+ kthread_stop(thread); -+} -+KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client); -+ -+/* Accumulate counters in the dump buffer */ -+static void accum_dump_buffer(void *dst, void *src, size_t dump_size) -+{ -+ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; -+ u32 *d = dst; -+ u32 *s = src; -+ size_t i, j; -+ -+ for (i = 0; i < dump_size; i += block_size) { -+ /* skip over the header block */ -+ d += NR_BYTES_PER_HDR / sizeof(u32); -+ s += NR_BYTES_PER_HDR / sizeof(u32); -+ for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) { -+ /* saturate result if addition would result in wraparound */ -+ if (U32_MAX - *d < *s) -+ *d = U32_MAX; -+ else -+ *d += *s; -+ d++; -+ s++; -+ } -+ } -+} -+ -+/* This is the Midgard v4 patch function. It copies the headers for each -+ * of the defined blocks from the master kernel buffer and then patches up -+ * the performance counter enable mask for each of the blocks to exclude -+ * counters that were not requested by the client. */ -+static void patch_dump_buffer_hdr_v4( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_vinstr_client *cli) -+{ -+ u32 *mask; -+ u8 *dst = cli->accum_buffer; -+ u8 *src = vinstr_ctx->cpu_va; -+ u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups; -+ size_t i, group_size, group; -+ enum { -+ SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, -+ SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, -+ SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, -+ SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, -+ TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, -+ MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT, -+ JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT -+ }; -+ -+ group_size = NR_CNT_BLOCKS_PER_GROUP * -+ NR_CNT_PER_BLOCK * -+ NR_BYTES_PER_CNT; -+ for (i = 0; i < nr_cg; i++) { -+ group = i * group_size; -+ /* copy shader core headers */ -+ memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE], -+ NR_BYTES_PER_HDR); -+ memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE], -+ NR_BYTES_PER_HDR); -+ memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE], -+ NR_BYTES_PER_HDR); -+ memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE], -+ NR_BYTES_PER_HDR); -+ -+ /* copy tiler header */ -+ memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE], -+ NR_BYTES_PER_HDR); -+ -+ /* copy mmu header */ -+ memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE], -+ NR_BYTES_PER_HDR); -+ -+ /* copy job manager header */ -+ memcpy(&dst[group + JM_BASE], &src[group + JM_BASE], -+ NR_BYTES_PER_HDR); -+ -+ /* patch the shader core enable mask */ -+ mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[SHADER_HWCNT_BM]; -+ mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[SHADER_HWCNT_BM]; -+ mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[SHADER_HWCNT_BM]; -+ mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[SHADER_HWCNT_BM]; -+ -+ /* patch the tiler core enable mask */ -+ mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[TILER_HWCNT_BM]; -+ -+ /* patch the mmu core enable mask */ -+ mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; -+ -+ /* patch the job manager enable mask */ -+ mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[JM_HWCNT_BM]; -+ } -+} -+ -+/* This is the Midgard v5 patch function. It copies the headers for each -+ * of the defined blocks from the master kernel buffer and then patches up -+ * the performance counter enable mask for each of the blocks to exclude -+ * counters that were not requested by the client. */ -+static void patch_dump_buffer_hdr_v5( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_vinstr_client *cli) -+{ -+ struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev; -+ u32 i, nr_l2; -+ u64 core_mask; -+ u32 *mask; -+ u8 *dst = cli->accum_buffer; -+ u8 *src = vinstr_ctx->cpu_va; -+ size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT; -+ -+ /* copy and patch job manager header */ -+ memcpy(dst, src, NR_BYTES_PER_HDR); -+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[JM_HWCNT_BM]; -+ dst += block_size; -+ src += block_size; -+ -+ /* copy and patch tiler header */ -+ memcpy(dst, src, NR_BYTES_PER_HDR); -+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[TILER_HWCNT_BM]; -+ dst += block_size; -+ src += block_size; -+ -+ /* copy and patch MMU/L2C headers */ -+ nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices; -+ for (i = 0; i < nr_l2; i++) { -+ memcpy(dst, src, NR_BYTES_PER_HDR); -+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[MMU_L2_HWCNT_BM]; -+ dst += block_size; -+ src += block_size; -+ } -+ -+ /* copy and patch shader core headers */ -+ core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; -+ while (0ull != core_mask) { -+ memcpy(dst, src, NR_BYTES_PER_HDR); -+ if (0ull != (core_mask & 1ull)) { -+ /* if block is not reserved update header */ -+ mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET]; -+ *mask &= cli->bitmap[SHADER_HWCNT_BM]; -+ } -+ dst += block_size; -+ src += block_size; -+ -+ core_mask >>= 1; -+ } -+} -+ -+/** -+ * accum_clients - accumulate dumped hw counters for all known clients -+ * @vinstr_ctx: vinstr context -+ */ -+static void accum_clients(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_vinstr_client *iter; -+ int v4 = 0; -+ -+#ifndef CONFIG_MALI_NO_MALI -+ v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4); -+#endif -+ -+ list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) { -+ /* Don't bother accumulating clients whose hwcnt requests -+ * have not yet been honoured. */ -+ if (iter->pending) -+ continue; -+ if (v4) -+ patch_dump_buffer_hdr_v4(vinstr_ctx, iter); -+ else -+ patch_dump_buffer_hdr_v5(vinstr_ctx, iter); -+ accum_dump_buffer( -+ iter->accum_buffer, -+ vinstr_ctx->cpu_va, -+ iter->dump_size); -+ } -+ list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) { -+ /* Don't bother accumulating clients whose hwcnt requests -+ * have not yet been honoured. */ -+ if (iter->pending) -+ continue; -+ if (v4) -+ patch_dump_buffer_hdr_v4(vinstr_ctx, iter); -+ else -+ patch_dump_buffer_hdr_v5(vinstr_ctx, iter); -+ accum_dump_buffer( -+ iter->accum_buffer, -+ vinstr_ctx->cpu_va, -+ iter->dump_size); -+ } -+} -+ -+/*****************************************************************************/ -+ -+/** -+ * kbasep_vinstr_get_timestamp - return timestamp -+ * -+ * Function returns timestamp value based on raw monotonic timer. Value will -+ * wrap around zero in case of overflow. -+ * -+ * Return: timestamp value -+ */ -+static u64 kbasep_vinstr_get_timestamp(void) -+{ -+ struct timespec64 ts; -+ -+ ktime_get_raw_ts64(&ts); -+ return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec; -+} -+ -+/** -+ * kbasep_vinstr_add_dump_request - register client's dumping request -+ * @cli: requesting client -+ * @waiting_clients: list of pending dumping requests -+ */ -+static void kbasep_vinstr_add_dump_request( -+ struct kbase_vinstr_client *cli, -+ struct list_head *waiting_clients) -+{ -+ struct kbase_vinstr_client *tmp; -+ -+ if (list_empty(waiting_clients)) { -+ list_add(&cli->list, waiting_clients); -+ return; -+ } -+ list_for_each_entry(tmp, waiting_clients, list) { -+ if (tmp->dump_time > cli->dump_time) { -+ list_add_tail(&cli->list, &tmp->list); -+ return; -+ } -+ } -+ list_add_tail(&cli->list, waiting_clients); -+} -+ -+/** -+ * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level -+ * dump and accumulate them for known -+ * clients -+ * @vinstr_ctx: vinstr context -+ * @timestamp: pointer where collection timestamp will be recorded -+ * -+ * Return: zero on success -+ */ -+static int kbasep_vinstr_collect_and_accumulate( -+ struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp) -+{ -+ unsigned long flags; -+ int rcode; -+ -+#ifdef CONFIG_MALI_NO_MALI -+ /* The dummy model needs the CPU mapping. */ -+ gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va); -+#endif -+ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ if (VINSTR_IDLE != vinstr_ctx->state) { -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ return -EAGAIN; -+ } else { -+ vinstr_ctx->state = VINSTR_DUMPING; -+ } -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ -+ /* Request HW counters dump. -+ * Disable preemption to make dump timestamp more accurate. */ -+ preempt_disable(); -+ *timestamp = kbasep_vinstr_get_timestamp(); -+ rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx); -+ preempt_enable(); -+ -+ if (!rcode) -+ rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx); -+ WARN_ON(rcode); -+ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ switch (vinstr_ctx->state) -+ { -+ case VINSTR_SUSPENDING: -+ schedule_work(&vinstr_ctx->suspend_work); -+ break; -+ case VINSTR_DUMPING: -+ vinstr_ctx->state = VINSTR_IDLE; -+ wake_up_all(&vinstr_ctx->suspend_waitq); -+ break; -+ default: -+ break; -+ } -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ -+ /* Accumulate values of collected counters. */ -+ if (!rcode) -+ accum_clients(vinstr_ctx); -+ -+ return rcode; -+} -+ -+/** -+ * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel -+ * buffer -+ * @cli: requesting client -+ * @timestamp: timestamp when counters were collected -+ * @event_id: id of event that caused triggered counters collection -+ * -+ * Return: zero on success -+ */ -+static int kbasep_vinstr_fill_dump_buffer( -+ struct kbase_vinstr_client *cli, u64 timestamp, -+ enum base_hwcnt_reader_event event_id) -+{ -+ unsigned int write_idx = atomic_read(&cli->write_idx); -+ unsigned int read_idx = atomic_read(&cli->read_idx); -+ -+ struct kbase_hwcnt_reader_metadata *meta; -+ void *buffer; -+ -+ /* Check if there is a place to copy HWC block into. */ -+ if (write_idx - read_idx == cli->buffer_count) -+ return -1; -+ write_idx %= cli->buffer_count; -+ -+ /* Fill in dump buffer and its metadata. */ -+ buffer = &cli->dump_buffers[write_idx * cli->dump_size]; -+ meta = &cli->dump_buffers_meta[write_idx]; -+ meta->timestamp = timestamp; -+ meta->event_id = event_id; -+ meta->buffer_idx = write_idx; -+ memcpy(buffer, cli->accum_buffer, cli->dump_size); -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer -+ * allocated in userspace -+ * @cli: requesting client -+ * -+ * Return: zero on success -+ * -+ * This is part of legacy ioctl interface. -+ */ -+static int kbasep_vinstr_fill_dump_buffer_legacy( -+ struct kbase_vinstr_client *cli) -+{ -+ void __user *buffer = cli->legacy_buffer; -+ int rcode; -+ -+ /* Copy data to user buffer. */ -+ rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size); -+ if (rcode) -+ pr_warn("error while copying buffer to user\n"); -+ return rcode; -+} -+ -+/** -+ * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer -+ * allocated in kernel space -+ * @cli: requesting client -+ * -+ * Return: zero on success -+ * -+ * This is part of the kernel client interface. -+ */ -+static int kbasep_vinstr_fill_dump_buffer_kernel( -+ struct kbase_vinstr_client *cli) -+{ -+ memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst -+ * @vinstr_ctx: vinstr context -+ */ -+static void kbasep_vinstr_reprogram( -+ struct kbase_vinstr_context *vinstr_ctx) -+{ -+ unsigned long flags; -+ bool suspended = false; -+ -+ /* Don't enable hardware counters if vinstr is suspended. */ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ if (VINSTR_IDLE != vinstr_ctx->state) -+ suspended = true; -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ if (suspended) -+ return; -+ -+ /* Change to suspended state is done while holding vinstr context -+ * lock. Below code will then no re-enable the instrumentation. */ -+ -+ if (vinstr_ctx->reprogram) { -+ struct kbase_vinstr_client *iter; -+ -+ if (!reprogram_hwcnt(vinstr_ctx)) { -+ vinstr_ctx->reprogram = false; -+ list_for_each_entry( -+ iter, -+ &vinstr_ctx->idle_clients, -+ list) -+ iter->pending = false; -+ list_for_each_entry( -+ iter, -+ &vinstr_ctx->waiting_clients, -+ list) -+ iter->pending = false; -+ } -+ } -+} -+ -+/** -+ * kbasep_vinstr_update_client - copy accumulated counters to user readable -+ * buffer and notify the user -+ * @cli: requesting client -+ * @timestamp: timestamp when counters were collected -+ * @event_id: id of event that caused triggered counters collection -+ * -+ * Return: zero on success -+ */ -+static int kbasep_vinstr_update_client( -+ struct kbase_vinstr_client *cli, u64 timestamp, -+ enum base_hwcnt_reader_event event_id) -+{ -+ int rcode = 0; -+ -+ /* Copy collected counters to user readable buffer. */ -+ if (cli->buffer_count) -+ rcode = kbasep_vinstr_fill_dump_buffer( -+ cli, timestamp, event_id); -+ else if (cli->kernel_buffer) -+ rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli); -+ else -+ rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli); -+ -+ if (rcode) -+ goto exit; -+ -+ -+ /* Notify client. Make sure all changes to memory are visible. */ -+ wmb(); -+ atomic_inc(&cli->write_idx); -+ wake_up_interruptible(&cli->waitq); -+ -+ /* Prepare for next request. */ -+ memset(cli->accum_buffer, 0, cli->dump_size); -+ -+exit: -+ return rcode; -+} -+ -+/** -+ * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function -+ * -+ * @hrtimer: high resolution timer -+ * -+ * Return: High resolution timer restart enum. -+ */ -+static enum hrtimer_restart kbasep_vinstr_wake_up_callback( -+ struct hrtimer *hrtimer) -+{ -+ struct kbasep_vinstr_wake_up_timer *timer = -+ container_of( -+ hrtimer, -+ struct kbasep_vinstr_wake_up_timer, -+ hrtimer); -+ -+ KBASE_DEBUG_ASSERT(timer); -+ -+ atomic_set(&timer->vinstr_ctx->request_pending, 1); -+ wake_up_all(&timer->vinstr_ctx->waitq); -+ -+ return HRTIMER_NORESTART; -+} -+ -+#ifdef CONFIG_DEBUG_OBJECT_TIMERS -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) -+/** -+ * kbase_destroy_hrtimer_on_stack - kernel's destroy_hrtimer_on_stack(), -+ * rewritten -+ * -+ * @timer: high resolution timer -+ * -+ * destroy_hrtimer_on_stack() was exported only for 4.7.0 kernel so for -+ * earlier kernel versions it is not possible to call it explicitly. -+ * Since this function must accompany hrtimer_init_on_stack(), which -+ * has to be used for hrtimer initialization if CONFIG_DEBUG_OBJECT_TIMERS -+ * is defined in order to avoid the warning about object on stack not being -+ * annotated, we rewrite it here to be used for earlier kernel versions. -+ */ -+static void kbase_destroy_hrtimer_on_stack(struct hrtimer *timer) -+{ -+ debug_object_free(timer, &hrtimer_debug_descr); -+} -+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) */ -+#endif /* CONFIG_DEBUG_OBJECT_TIMERS */ -+ -+/** -+ * kbasep_vinstr_service_task - HWC dumping service thread -+ * -+ * @data: Pointer to vinstr context structure. -+ * -+ * Return: Always returns zero. -+ */ -+static int kbasep_vinstr_service_task(void *data) -+{ -+ struct kbase_vinstr_context *vinstr_ctx = data; -+ struct kbasep_vinstr_wake_up_timer timer; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ hrtimer_init_on_stack(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ -+ timer.hrtimer.function = kbasep_vinstr_wake_up_callback; -+ timer.vinstr_ctx = vinstr_ctx; -+ -+ while (!kthread_should_stop()) { -+ struct kbase_vinstr_client *cli = NULL; -+ struct kbase_vinstr_client *tmp; -+ int rcode; -+ -+ u64 timestamp = kbasep_vinstr_get_timestamp(); -+ u64 dump_time = 0; -+ struct list_head expired_requests; -+ -+ /* Hold lock while performing operations on lists of clients. */ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ /* Closing thread must not interact with client requests. */ -+ if (current == vinstr_ctx->thread) { -+ atomic_set(&vinstr_ctx->request_pending, 0); -+ -+ if (!list_empty(&vinstr_ctx->waiting_clients)) { -+ cli = list_first_entry( -+ &vinstr_ctx->waiting_clients, -+ struct kbase_vinstr_client, -+ list); -+ dump_time = cli->dump_time; -+ } -+ } -+ -+ if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) { -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ /* Sleep until next dumping event or service request. */ -+ if (cli) { -+ u64 diff = dump_time - timestamp; -+ -+ hrtimer_start( -+ &timer.hrtimer, -+ ns_to_ktime(diff), -+ HRTIMER_MODE_REL); -+ } -+ wait_event( -+ vinstr_ctx->waitq, -+ atomic_read( -+ &vinstr_ctx->request_pending) || -+ kthread_should_stop()); -+ hrtimer_cancel(&timer.hrtimer); -+ continue; -+ } -+ -+ rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, -+ ×tamp); -+ -+ INIT_LIST_HEAD(&expired_requests); -+ -+ /* Find all expired requests. */ -+ list_for_each_entry_safe( -+ cli, -+ tmp, -+ &vinstr_ctx->waiting_clients, -+ list) { -+ s64 tdiff = -+ (s64)(timestamp + DUMPING_RESOLUTION) - -+ (s64)cli->dump_time; -+ if (tdiff >= 0ll) { -+ list_del(&cli->list); -+ list_add(&cli->list, &expired_requests); -+ } else { -+ break; -+ } -+ } -+ -+ /* Fill data for each request found. */ -+ list_for_each_entry_safe(cli, tmp, &expired_requests, list) { -+ /* Ensure that legacy buffer will not be used from -+ * this kthread context. */ -+ BUG_ON(0 == cli->buffer_count); -+ /* Expect only periodically sampled clients. */ -+ BUG_ON(0 == cli->dump_interval); -+ -+ if (!rcode) -+ kbasep_vinstr_update_client( -+ cli, -+ timestamp, -+ BASE_HWCNT_READER_EVENT_PERIODIC); -+ -+ /* Set new dumping time. Drop missed probing times. */ -+ do { -+ cli->dump_time += cli->dump_interval; -+ } while (cli->dump_time < timestamp); -+ -+ list_del(&cli->list); -+ kbasep_vinstr_add_dump_request( -+ cli, -+ &vinstr_ctx->waiting_clients); -+ } -+ -+ /* Reprogram counters set if required. */ -+ kbasep_vinstr_reprogram(vinstr_ctx); -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ } -+ -+#ifdef CONFIG_DEBUG_OBJECTS_TIMERS -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) -+ kbase_destroy_hrtimer_on_stack(&timer.hrtimer); -+#else -+ destroy_hrtimer_on_stack(&timer.hrtimer); -+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)) */ -+#endif /* CONFIG_DEBUG_OBJECTS_TIMERS */ -+ -+ return 0; -+} -+ -+/*****************************************************************************/ -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers -+ * @cli: pointer to vinstr client structure -+ * -+ * Return: non-zero if client has at least one dumping buffer filled that was -+ * not notified to user yet -+ */ -+static int kbasep_vinstr_hwcnt_reader_buffer_ready( -+ struct kbase_vinstr_client *cli) -+{ -+ KBASE_DEBUG_ASSERT(cli); -+ return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command -+ * @cli: pointer to vinstr client structure -+ * @buffer: pointer to userspace buffer -+ * @size: size of buffer -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( -+ struct kbase_vinstr_client *cli, void __user *buffer, -+ size_t size) -+{ -+ unsigned int meta_idx = atomic_read(&cli->meta_idx); -+ unsigned int idx = meta_idx % cli->buffer_count; -+ -+ struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx]; -+ -+ /* Metadata sanity check. */ -+ KBASE_DEBUG_ASSERT(idx == meta->buffer_idx); -+ -+ if (sizeof(struct kbase_hwcnt_reader_metadata) != size) -+ return -EINVAL; -+ -+ /* Check if there is any buffer available. */ -+ if (atomic_read(&cli->write_idx) == meta_idx) -+ return -EAGAIN; -+ -+ /* Check if previously taken buffer was put back. */ -+ if (atomic_read(&cli->read_idx) != meta_idx) -+ return -EBUSY; -+ -+ /* Copy next available buffer's metadata to user. */ -+ if (copy_to_user(buffer, meta, size)) -+ return -EFAULT; -+ -+ atomic_inc(&cli->meta_idx); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command -+ * @cli: pointer to vinstr client structure -+ * @buffer: pointer to userspace buffer -+ * @size: size of buffer -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( -+ struct kbase_vinstr_client *cli, void __user *buffer, -+ size_t size) -+{ -+ unsigned int read_idx = atomic_read(&cli->read_idx); -+ unsigned int idx = read_idx % cli->buffer_count; -+ -+ struct kbase_hwcnt_reader_metadata meta; -+ -+ if (sizeof(struct kbase_hwcnt_reader_metadata) != size) -+ return -EINVAL; -+ -+ /* Check if any buffer was taken. */ -+ if (atomic_read(&cli->meta_idx) == read_idx) -+ return -EPERM; -+ -+ /* Check if correct buffer is put back. */ -+ if (copy_from_user(&meta, buffer, size)) -+ return -EFAULT; -+ if (idx != meta.buffer_idx) -+ return -EINVAL; -+ -+ atomic_inc(&cli->read_idx); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command -+ * @cli: pointer to vinstr client structure -+ * @interval: periodic dumping interval (disable periodic dumping if zero) -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( -+ struct kbase_vinstr_client *cli, u32 interval) -+{ -+ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ list_del(&cli->list); -+ -+ cli->dump_interval = interval; -+ -+ /* If interval is non-zero, enable periodic dumping for this client. */ -+ if (cli->dump_interval) { -+ if (DUMPING_RESOLUTION > cli->dump_interval) -+ cli->dump_interval = DUMPING_RESOLUTION; -+ cli->dump_time = -+ kbasep_vinstr_get_timestamp() + cli->dump_interval; -+ -+ kbasep_vinstr_add_dump_request( -+ cli, &vinstr_ctx->waiting_clients); -+ -+ atomic_set(&vinstr_ctx->request_pending, 1); -+ wake_up_all(&vinstr_ctx->waitq); -+ } else { -+ list_add(&cli->list, &vinstr_ctx->idle_clients); -+ } -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id -+ * @event_id: id of event -+ * Return: event_mask or zero if event is not supported or maskable -+ */ -+static u32 kbasep_vinstr_hwcnt_reader_event_mask( -+ enum base_hwcnt_reader_event event_id) -+{ -+ u32 event_mask = 0; -+ -+ switch (event_id) { -+ case BASE_HWCNT_READER_EVENT_PREJOB: -+ case BASE_HWCNT_READER_EVENT_POSTJOB: -+ /* These event are maskable. */ -+ event_mask = (1 << event_id); -+ break; -+ -+ case BASE_HWCNT_READER_EVENT_MANUAL: -+ case BASE_HWCNT_READER_EVENT_PERIODIC: -+ /* These event are non-maskable. */ -+ default: -+ /* These event are not supported. */ -+ break; -+ } -+ -+ return event_mask; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command -+ * @cli: pointer to vinstr client structure -+ * @event_id: id of event to enable -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event( -+ struct kbase_vinstr_client *cli, -+ enum base_hwcnt_reader_event event_id) -+{ -+ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; -+ u32 event_mask; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); -+ if (!event_mask) -+ return -EINVAL; -+ -+ mutex_lock(&vinstr_ctx->lock); -+ cli->event_mask |= event_mask; -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command -+ * @cli: pointer to vinstr client structure -+ * @event_id: id of event to disable -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event( -+ struct kbase_vinstr_client *cli, -+ enum base_hwcnt_reader_event event_id) -+{ -+ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; -+ u32 event_mask; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id); -+ if (!event_mask) -+ return -EINVAL; -+ -+ mutex_lock(&vinstr_ctx->lock); -+ cli->event_mask &= ~event_mask; -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command -+ * @cli: pointer to vinstr client structure -+ * @hwver: pointer to user buffer where hw version will be stored -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( -+ struct kbase_vinstr_client *cli, u32 __user *hwver) -+{ -+#ifndef CONFIG_MALI_NO_MALI -+ struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx; -+#endif -+ -+ u32 ver = 5; -+ -+#ifndef CONFIG_MALI_NO_MALI -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4)) -+ ver = 4; -+#endif -+ -+ return put_user(ver, hwver); -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl -+ * @filp: pointer to file structure -+ * @cmd: user command -+ * @arg: command's argument -+ * -+ * Return: zero on success -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp, -+ unsigned int cmd, unsigned long arg) -+{ -+ long rcode = 0; -+ struct kbase_vinstr_client *cli; -+ -+ KBASE_DEBUG_ASSERT(filp); -+ -+ cli = filp->private_data; -+ KBASE_DEBUG_ASSERT(cli); -+ -+ if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd))) -+ return -EINVAL; -+ -+ switch (cmd) { -+ case KBASE_HWCNT_READER_GET_API_VERSION: -+ rcode = put_user(HWCNT_READER_API, (u32 __user *)arg); -+ break; -+ case KBASE_HWCNT_READER_GET_HWVER: -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( -+ cli, (u32 __user *)arg); -+ break; -+ case KBASE_HWCNT_READER_GET_BUFFER_SIZE: -+ KBASE_DEBUG_ASSERT(cli->vinstr_ctx); -+ rcode = put_user( -+ (u32)cli->vinstr_ctx->dump_size, -+ (u32 __user *)arg); -+ break; -+ case KBASE_HWCNT_READER_DUMP: -+ rcode = kbase_vinstr_hwc_dump( -+ cli, BASE_HWCNT_READER_EVENT_MANUAL); -+ break; -+ case KBASE_HWCNT_READER_CLEAR: -+ rcode = kbase_vinstr_hwc_clear(cli); -+ break; -+ case KBASE_HWCNT_READER_GET_BUFFER: -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( -+ cli, (void __user *)arg, _IOC_SIZE(cmd)); -+ break; -+ case KBASE_HWCNT_READER_PUT_BUFFER: -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( -+ cli, (void __user *)arg, _IOC_SIZE(cmd)); -+ break; -+ case KBASE_HWCNT_READER_SET_INTERVAL: -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( -+ cli, (u32)arg); -+ break; -+ case KBASE_HWCNT_READER_ENABLE_EVENT: -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( -+ cli, (enum base_hwcnt_reader_event)arg); -+ break; -+ case KBASE_HWCNT_READER_DISABLE_EVENT: -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( -+ cli, (enum base_hwcnt_reader_event)arg); -+ break; -+ default: -+ rcode = -EINVAL; -+ break; -+ } -+ -+ return rcode; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll -+ * @filp: pointer to file structure -+ * @wait: pointer to poll table -+ * Return: POLLIN if data can be read without blocking, otherwise zero -+ */ -+static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp, -+ poll_table *wait) -+{ -+ struct kbase_vinstr_client *cli; -+ -+ KBASE_DEBUG_ASSERT(filp); -+ KBASE_DEBUG_ASSERT(wait); -+ -+ cli = filp->private_data; -+ KBASE_DEBUG_ASSERT(cli); -+ -+ poll_wait(filp, &cli->waitq, wait); -+ if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli)) -+ return POLLIN; -+ return 0; -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap -+ * @filp: pointer to file structure -+ * @vma: pointer to vma structure -+ * Return: zero on success -+ */ -+static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp, -+ struct vm_area_struct *vma) -+{ -+ struct kbase_vinstr_client *cli; -+ unsigned long size, addr, pfn, offset; -+ unsigned long vm_size = vma->vm_end - vma->vm_start; -+ -+ KBASE_DEBUG_ASSERT(filp); -+ KBASE_DEBUG_ASSERT(vma); -+ -+ cli = filp->private_data; -+ KBASE_DEBUG_ASSERT(cli); -+ -+ size = cli->buffer_count * cli->dump_size; -+ -+ if (vma->vm_pgoff > (size >> PAGE_SHIFT)) -+ return -EINVAL; -+ -+ offset = vma->vm_pgoff << PAGE_SHIFT; -+ if (vm_size > size - offset) -+ return -EINVAL; -+ -+ addr = __pa((unsigned long)cli->dump_buffers + offset); -+ pfn = addr >> PAGE_SHIFT; -+ -+ return remap_pfn_range( -+ vma, -+ vma->vm_start, -+ pfn, -+ vm_size, -+ vma->vm_page_prot); -+} -+ -+/** -+ * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release -+ * @inode: pointer to inode structure -+ * @filp: pointer to file structure -+ * Return always return zero -+ */ -+static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode, -+ struct file *filp) -+{ -+ struct kbase_vinstr_client *cli; -+ -+ KBASE_DEBUG_ASSERT(inode); -+ KBASE_DEBUG_ASSERT(filp); -+ -+ cli = filp->private_data; -+ KBASE_DEBUG_ASSERT(cli); -+ -+ kbase_vinstr_detach_client(cli); -+ return 0; -+} -+ -+/*****************************************************************************/ -+ -+/** -+ * kbasep_vinstr_kick_scheduler - trigger scheduler cycle -+ * @kbdev: pointer to kbase device structure -+ */ -+static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev) -+{ -+ struct kbasep_js_device_data *js_devdata = &kbdev->js_data; -+ unsigned long flags; -+ -+ down(&js_devdata->schedule_sem); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_backend_slot_update(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ up(&js_devdata->schedule_sem); -+} -+ -+/** -+ * kbasep_vinstr_suspend_worker - worker suspending vinstr module -+ * @data: pointer to work structure -+ */ -+static void kbasep_vinstr_suspend_worker(struct work_struct *data) -+{ -+ struct kbase_vinstr_context *vinstr_ctx; -+ unsigned long flags; -+ -+ vinstr_ctx = container_of(data, struct kbase_vinstr_context, -+ suspend_work); -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ if (vinstr_ctx->kctx) -+ disable_hwcnt(vinstr_ctx); -+ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ vinstr_ctx->state = VINSTR_SUSPENDED; -+ wake_up_all(&vinstr_ctx->suspend_waitq); -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ /* Kick GPU scheduler to allow entering protected mode. -+ * This must happen after vinstr was suspended. */ -+ kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); -+} -+ -+/** -+ * kbasep_vinstr_suspend_worker - worker resuming vinstr module -+ * @data: pointer to work structure -+ */ -+static void kbasep_vinstr_resume_worker(struct work_struct *data) -+{ -+ struct kbase_vinstr_context *vinstr_ctx; -+ unsigned long flags; -+ -+ vinstr_ctx = container_of(data, struct kbase_vinstr_context, -+ resume_work); -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ if (vinstr_ctx->kctx) -+ enable_hwcnt(vinstr_ctx); -+ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ vinstr_ctx->state = VINSTR_IDLE; -+ wake_up_all(&vinstr_ctx->suspend_waitq); -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ /* Kick GPU scheduler to allow entering protected mode. -+ * Note that scheduler state machine might requested re-entry to -+ * protected mode before vinstr was resumed. -+ * This must happen after vinstr was release. */ -+ kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev); -+} -+ -+/*****************************************************************************/ -+ -+struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev) -+{ -+ struct kbase_vinstr_context *vinstr_ctx; -+ -+ vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL); -+ if (!vinstr_ctx) -+ return NULL; -+ -+ INIT_LIST_HEAD(&vinstr_ctx->idle_clients); -+ INIT_LIST_HEAD(&vinstr_ctx->waiting_clients); -+ mutex_init(&vinstr_ctx->lock); -+ spin_lock_init(&vinstr_ctx->state_lock); -+ vinstr_ctx->kbdev = kbdev; -+ vinstr_ctx->thread = NULL; -+ vinstr_ctx->state = VINSTR_IDLE; -+ vinstr_ctx->suspend_cnt = 0; -+ INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker); -+ INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker); -+ init_waitqueue_head(&vinstr_ctx->suspend_waitq); -+ -+ atomic_set(&vinstr_ctx->request_pending, 0); -+ init_waitqueue_head(&vinstr_ctx->waitq); -+ -+ return vinstr_ctx; -+} -+ -+void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ struct kbase_vinstr_client *cli; -+ -+ /* Stop service thread first. */ -+ if (vinstr_ctx->thread) -+ kthread_stop(vinstr_ctx->thread); -+ -+ /* Wait for workers. */ -+ flush_work(&vinstr_ctx->suspend_work); -+ flush_work(&vinstr_ctx->resume_work); -+ -+ while (1) { -+ struct list_head *list = &vinstr_ctx->idle_clients; -+ -+ if (list_empty(list)) { -+ list = &vinstr_ctx->waiting_clients; -+ if (list_empty(list)) -+ break; -+ } -+ -+ cli = list_first_entry(list, struct kbase_vinstr_client, list); -+ list_del(&cli->list); -+ kfree(cli->accum_buffer); -+ kfree(cli); -+ vinstr_ctx->nclients--; -+ } -+ KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients); -+ if (vinstr_ctx->kctx) -+ kbasep_vinstr_destroy_kctx(vinstr_ctx); -+ kfree(vinstr_ctx); -+} -+ -+int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_uk_hwcnt_reader_setup *setup) -+{ -+ struct kbase_vinstr_client *cli; -+ u32 bitmap[4]; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ KBASE_DEBUG_ASSERT(setup); -+ KBASE_DEBUG_ASSERT(setup->buffer_count); -+ -+ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; -+ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; -+ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; -+ bitmap[JM_HWCNT_BM] = setup->jm_bm; -+ -+ cli = kbasep_vinstr_attach_client( -+ vinstr_ctx, -+ setup->buffer_count, -+ bitmap, -+ &setup->fd, -+ NULL); -+ -+ if (!cli) -+ return -ENOMEM; -+ -+ return 0; -+} -+ -+int kbase_vinstr_legacy_hwc_setup( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_vinstr_client **cli, -+ struct kbase_uk_hwcnt_setup *setup) -+{ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ KBASE_DEBUG_ASSERT(setup); -+ KBASE_DEBUG_ASSERT(cli); -+ -+ if (setup->dump_buffer) { -+ u32 bitmap[4]; -+ -+ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; -+ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; -+ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; -+ bitmap[JM_HWCNT_BM] = setup->jm_bm; -+ -+ if (*cli) -+ return -EBUSY; -+ -+ *cli = kbasep_vinstr_attach_client( -+ vinstr_ctx, -+ 0, -+ bitmap, -+ (void *)(long)setup->dump_buffer, -+ NULL); -+ -+ if (!(*cli)) -+ return -ENOMEM; -+ } else { -+ if (!*cli) -+ return -EINVAL; -+ -+ kbase_vinstr_detach_client(*cli); -+ *cli = NULL; -+ } -+ -+ return 0; -+} -+ -+struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_uk_hwcnt_reader_setup *setup, -+ void *kernel_buffer) -+{ -+ u32 bitmap[4]; -+ -+ if (!vinstr_ctx || !setup || !kernel_buffer) -+ return NULL; -+ -+ bitmap[SHADER_HWCNT_BM] = setup->shader_bm; -+ bitmap[TILER_HWCNT_BM] = setup->tiler_bm; -+ bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm; -+ bitmap[JM_HWCNT_BM] = setup->jm_bm; -+ -+ return kbasep_vinstr_attach_client( -+ vinstr_ctx, -+ 0, -+ bitmap, -+ NULL, -+ kernel_buffer); -+} -+KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup); -+ -+int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli, -+ enum base_hwcnt_reader_event event_id) -+{ -+ int rcode = 0; -+ struct kbase_vinstr_context *vinstr_ctx; -+ u64 timestamp; -+ u32 event_mask; -+ -+ if (!cli) -+ return -EINVAL; -+ -+ vinstr_ctx = cli->vinstr_ctx; -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT); -+ event_mask = 1 << event_id; -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ if (event_mask & cli->event_mask) { -+ rcode = kbasep_vinstr_collect_and_accumulate( -+ vinstr_ctx, -+ ×tamp); -+ if (rcode) -+ goto exit; -+ -+ rcode = kbasep_vinstr_update_client(cli, timestamp, event_id); -+ if (rcode) -+ goto exit; -+ -+ kbasep_vinstr_reprogram(vinstr_ctx); -+ } -+ -+exit: -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ return rcode; -+} -+KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump); -+ -+int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli) -+{ -+ struct kbase_vinstr_context *vinstr_ctx; -+ int rcode; -+ u64 unused; -+ -+ if (!cli) -+ return -EINVAL; -+ -+ vinstr_ctx = cli->vinstr_ctx; -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ mutex_lock(&vinstr_ctx->lock); -+ -+ rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused); -+ if (rcode) -+ goto exit; -+ rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx); -+ if (rcode) -+ goto exit; -+ memset(cli->accum_buffer, 0, cli->dump_size); -+ -+ kbasep_vinstr_reprogram(vinstr_ctx); -+ -+exit: -+ mutex_unlock(&vinstr_ctx->lock); -+ -+ return rcode; -+} -+ -+int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ unsigned long flags; -+ int ret = -EAGAIN; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ switch (vinstr_ctx->state) { -+ case VINSTR_SUSPENDED: -+ vinstr_ctx->suspend_cnt++; -+ /* overflow shall not happen */ -+ BUG_ON(0 == vinstr_ctx->suspend_cnt); -+ ret = 0; -+ break; -+ -+ case VINSTR_IDLE: -+ vinstr_ctx->state = VINSTR_SUSPENDING; -+ schedule_work(&vinstr_ctx->suspend_work); -+ break; -+ -+ case VINSTR_DUMPING: -+ vinstr_ctx->state = VINSTR_SUSPENDING; -+ break; -+ -+ case VINSTR_SUSPENDING: -+ /* fall through */ -+ case VINSTR_RESUMING: -+ break; -+ -+ default: -+ BUG(); -+ break; -+ } -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+ -+ return ret; -+} -+ -+void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ wait_event(vinstr_ctx->suspend_waitq, -+ (0 == kbase_vinstr_try_suspend(vinstr_ctx))); -+} -+ -+void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx) -+{ -+ unsigned long flags; -+ -+ KBASE_DEBUG_ASSERT(vinstr_ctx); -+ -+ spin_lock_irqsave(&vinstr_ctx->state_lock, flags); -+ BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state); -+ if (VINSTR_SUSPENDED == vinstr_ctx->state) { -+ BUG_ON(0 == vinstr_ctx->suspend_cnt); -+ vinstr_ctx->suspend_cnt--; -+ if (0 == vinstr_ctx->suspend_cnt) { -+ vinstr_ctx->state = VINSTR_RESUMING; -+ schedule_work(&vinstr_ctx->resume_work); -+ } -+ } -+ spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags); -+} -diff --git a/drivers/gpu/arm/midgard/mali_kbase_vinstr.h b/drivers/gpu/arm/midgard/mali_kbase_vinstr.h -new file mode 100755 -index 000000000..6207d25ae ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_kbase_vinstr.h -@@ -0,0 +1,155 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KBASE_VINSTR_H_ -+#define _KBASE_VINSTR_H_ -+ -+#include -+#include -+ -+/*****************************************************************************/ -+ -+struct kbase_vinstr_context; -+struct kbase_vinstr_client; -+ -+/*****************************************************************************/ -+ -+/** -+ * kbase_vinstr_init() - initialize the vinstr core -+ * @kbdev: kbase device -+ * -+ * Return: pointer to the vinstr context on success or NULL on failure -+ */ -+struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_vinstr_term() - terminate the vinstr core -+ * @vinstr_ctx: vinstr context -+ */ -+void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx); -+ -+/** -+ * kbase_vinstr_hwcnt_reader_setup - configure hw counters reader -+ * @vinstr_ctx: vinstr context -+ * @setup: reader's configuration -+ * -+ * Return: zero on success -+ */ -+int kbase_vinstr_hwcnt_reader_setup( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_uk_hwcnt_reader_setup *setup); -+ -+/** -+ * kbase_vinstr_legacy_hwc_setup - configure hw counters for dumping -+ * @vinstr_ctx: vinstr context -+ * @cli: pointer where to store pointer to new vinstr client structure -+ * @setup: hwc configuration -+ * -+ * Return: zero on success -+ */ -+int kbase_vinstr_legacy_hwc_setup( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_vinstr_client **cli, -+ struct kbase_uk_hwcnt_setup *setup); -+ -+/** -+ * kbase_vinstr_hwcnt_kernel_setup - configure hw counters for kernel side -+ * client -+ * @vinstr_ctx: vinstr context -+ * @setup: reader's configuration -+ * @kernel_buffer: pointer to dump buffer -+ * -+ * setup->buffer_count and setup->fd are not used for kernel side clients. -+ * -+ * Return: pointer to client structure, or NULL on failure -+ */ -+struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup( -+ struct kbase_vinstr_context *vinstr_ctx, -+ struct kbase_uk_hwcnt_reader_setup *setup, -+ void *kernel_buffer); -+ -+/** -+ * kbase_vinstr_hwc_dump - issue counter dump for vinstr client -+ * @cli: pointer to vinstr client -+ * @event_id: id of event that triggered hwcnt dump -+ * -+ * Return: zero on success -+ */ -+int kbase_vinstr_hwc_dump( -+ struct kbase_vinstr_client *cli, -+ enum base_hwcnt_reader_event event_id); -+ -+/** -+ * kbase_vinstr_hwc_clear - performs a reset of the hardware counters for -+ * a given kbase context -+ * @cli: pointer to vinstr client -+ * -+ * Return: zero on success -+ */ -+int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli); -+ -+/** -+ * kbase_vinstr_try_suspend - try suspending operation of a given vinstr context -+ * @vinstr_ctx: vinstr context -+ * -+ * Return: 0 on success, or negative if state change is in progress -+ * -+ * Warning: This API call is non-generic. It is meant to be used only by -+ * job scheduler state machine. -+ * -+ * Function initiates vinstr switch to suspended state. Once it was called -+ * vinstr enters suspending state. If function return non-zero value, it -+ * indicates that state switch is not complete and function must be called -+ * again. On state switch vinstr will trigger job scheduler state machine -+ * cycle. -+ */ -+int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx); -+ -+/** -+ * kbase_vinstr_suspend - suspends operation of a given vinstr context -+ * @vinstr_ctx: vinstr context -+ * -+ * Function initiates vinstr switch to suspended state. Then it blocks until -+ * operation is completed. -+ */ -+void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx); -+ -+/** -+ * kbase_vinstr_resume - resumes operation of a given vinstr context -+ * @vinstr_ctx: vinstr context -+ * -+ * Function can be called only if it was preceded by a successful call -+ * to kbase_vinstr_suspend. -+ */ -+void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx); -+ -+/** -+ * kbase_vinstr_dump_size - Return required size of dump buffer -+ * @kbdev: device pointer -+ * -+ * Return : buffer size in bytes -+ */ -+size_t kbase_vinstr_dump_size(struct kbase_device *kbdev); -+ -+/** -+ * kbase_vinstr_detach_client - Detach a client from the vinstr core -+ * @cli: pointer to vinstr client -+ */ -+void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli); -+ -+#endif /* _KBASE_VINSTR_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h b/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h -new file mode 100755 -index 000000000..5d6b4021d ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_linux_kbase_trace.h -@@ -0,0 +1,201 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+#if !defined(_TRACE_MALI_KBASE_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_MALI_KBASE_H -+ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM mali -+ -+#include -+ -+DECLARE_EVENT_CLASS(mali_slot_template, -+ TP_PROTO(int jobslot, unsigned int info_val), -+ TP_ARGS(jobslot, info_val), -+ TP_STRUCT__entry( -+ __field(unsigned int, jobslot) -+ __field(unsigned int, info_val) -+ ), -+ TP_fast_assign( -+ __entry->jobslot = jobslot; -+ __entry->info_val = info_val; -+ ), -+ TP_printk("jobslot=%u info=%u", __entry->jobslot, __entry->info_val) -+); -+ -+#define DEFINE_MALI_SLOT_EVENT(name) \ -+DEFINE_EVENT(mali_slot_template, mali_##name, \ -+ TP_PROTO(int jobslot, unsigned int info_val), \ -+ TP_ARGS(jobslot, info_val)) -+DEFINE_MALI_SLOT_EVENT(JM_SUBMIT); -+DEFINE_MALI_SLOT_EVENT(JM_JOB_DONE); -+DEFINE_MALI_SLOT_EVENT(JM_UPDATE_HEAD); -+DEFINE_MALI_SLOT_EVENT(JM_CHECK_HEAD); -+DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP); -+DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_0); -+DEFINE_MALI_SLOT_EVENT(JM_SOFTSTOP_1); -+DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP); -+DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_0); -+DEFINE_MALI_SLOT_EVENT(JM_HARDSTOP_1); -+DEFINE_MALI_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); -+DEFINE_MALI_SLOT_EVENT(JM_SLOT_EVICT); -+DEFINE_MALI_SLOT_EVENT(JM_BEGIN_RESET_WORKER); -+DEFINE_MALI_SLOT_EVENT(JM_END_RESET_WORKER); -+DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); -+DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); -+DEFINE_MALI_SLOT_EVENT(JS_AFFINITY_CURRENT); -+DEFINE_MALI_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); -+DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); -+DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); -+DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); -+DEFINE_MALI_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); -+DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); -+DEFINE_MALI_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); -+DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); -+DEFINE_MALI_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); -+#undef DEFINE_MALI_SLOT_EVENT -+ -+DECLARE_EVENT_CLASS(mali_refcount_template, -+ TP_PROTO(int refcount, unsigned int info_val), -+ TP_ARGS(refcount, info_val), -+ TP_STRUCT__entry( -+ __field(unsigned int, refcount) -+ __field(unsigned int, info_val) -+ ), -+ TP_fast_assign( -+ __entry->refcount = refcount; -+ __entry->info_val = info_val; -+ ), -+ TP_printk("refcount=%u info=%u", __entry->refcount, __entry->info_val) -+); -+ -+#define DEFINE_MALI_REFCOUNT_EVENT(name) \ -+DEFINE_EVENT(mali_refcount_template, mali_##name, \ -+ TP_PROTO(int refcount, unsigned int info_val), \ -+ TP_ARGS(refcount, info_val)) -+DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX_NOLOCK); -+DEFINE_MALI_REFCOUNT_EVENT(JS_ADD_JOB); -+DEFINE_MALI_REFCOUNT_EVENT(JS_REMOVE_JOB); -+DEFINE_MALI_REFCOUNT_EVENT(JS_RETAIN_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_RELEASE_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); -+DEFINE_MALI_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); -+DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_ACTIVE); -+DEFINE_MALI_REFCOUNT_EVENT(PM_CONTEXT_IDLE); -+#undef DEFINE_MALI_REFCOUNT_EVENT -+ -+DECLARE_EVENT_CLASS(mali_add_template, -+ TP_PROTO(int gpu_addr, unsigned int info_val), -+ TP_ARGS(gpu_addr, info_val), -+ TP_STRUCT__entry( -+ __field(unsigned int, gpu_addr) -+ __field(unsigned int, info_val) -+ ), -+ TP_fast_assign( -+ __entry->gpu_addr = gpu_addr; -+ __entry->info_val = info_val; -+ ), -+ TP_printk("gpu_addr=%u info=%u", __entry->gpu_addr, __entry->info_val) -+); -+ -+#define DEFINE_MALI_ADD_EVENT(name) \ -+DEFINE_EVENT(mali_add_template, mali_##name, \ -+ TP_PROTO(int gpu_addr, unsigned int info_val), \ -+ TP_ARGS(gpu_addr, info_val)) -+DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); -+DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR); -+DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES); -+DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER); -+DEFINE_MALI_ADD_EVENT(JD_DONE_WORKER_END); -+DEFINE_MALI_ADD_EVENT(JD_CANCEL_WORKER); -+DEFINE_MALI_ADD_EVENT(JD_DONE); -+DEFINE_MALI_ADD_EVENT(JD_CANCEL); -+DEFINE_MALI_ADD_EVENT(JD_ZAP_CONTEXT); -+DEFINE_MALI_ADD_EVENT(JM_IRQ); -+DEFINE_MALI_ADD_EVENT(JM_IRQ_END); -+DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS); -+DEFINE_MALI_ADD_EVENT(JM_FLUSH_WORKQS_DONE); -+DEFINE_MALI_ADD_EVENT(JM_ZAP_NON_SCHEDULED); -+DEFINE_MALI_ADD_EVENT(JM_ZAP_SCHEDULED); -+DEFINE_MALI_ADD_EVENT(JM_ZAP_DONE); -+DEFINE_MALI_ADD_EVENT(JM_SUBMIT_AFTER_RESET); -+DEFINE_MALI_ADD_EVENT(JM_JOB_COMPLETE); -+DEFINE_MALI_ADD_EVENT(JS_FAST_START_EVICTS_CTX); -+DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); -+DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); -+DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); -+DEFINE_MALI_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); -+DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_END); -+DEFINE_MALI_ADD_EVENT(JS_POLICY_TIMER_START); -+DEFINE_MALI_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); -+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED); -+DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP); -+DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP); -+DEFINE_MALI_ADD_EVENT(PM_PWRON); -+DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER); -+DEFINE_MALI_ADD_EVENT(PM_PWRON_L2); -+DEFINE_MALI_ADD_EVENT(PM_PWROFF); -+DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER); -+DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2); -+DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED); -+DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER); -+DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2); -+DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED); -+DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER); -+DEFINE_MALI_ADD_EVENT(PM_UNREQUEST_CHANGE_SHADER_NEEDED); -+DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED); -+DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_NEEDED); -+DEFINE_MALI_ADD_EVENT(PM_REGISTER_CHANGE_SHADER_INUSE); -+DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_INUSE); -+DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); -+DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); -+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); -+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); -+DEFINE_MALI_ADD_EVENT(PM_GPU_ON); -+DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); -+DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); -+DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT); -+DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM); -+DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY); -+DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); -+#undef DEFINE_MALI_ADD_EVENT -+ -+#endif /* _TRACE_MALI_KBASE_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#undef linux -+#define TRACE_INCLUDE_PATH . -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE mali_linux_kbase_trace -+ -+/* This part must be outside protection */ -+#include -diff --git a/drivers/gpu/arm/midgard/mali_linux_trace.h b/drivers/gpu/arm/midgard/mali_linux_trace.h -new file mode 100755 -index 000000000..2be06a552 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_linux_trace.h -@@ -0,0 +1,189 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#if !defined(_TRACE_MALI_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_MALI_H -+ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM mali -+#define TRACE_INCLUDE_FILE mali_linux_trace -+ -+#include -+ -+#define MALI_JOB_SLOTS_EVENT_CHANGED -+ -+/** -+ * mali_job_slots_event - called from mali_kbase_core_linux.c -+ * @event_id: ORed together bitfields representing a type of event, made with the GATOR_MAKE_EVENT() macro. -+ */ -+TRACE_EVENT(mali_job_slots_event, -+ TP_PROTO(unsigned int event_id, unsigned int tgid, unsigned int pid, -+ unsigned char job_id), -+ TP_ARGS(event_id, tgid, pid, job_id), -+ TP_STRUCT__entry( -+ __field(unsigned int, event_id) -+ __field(unsigned int, tgid) -+ __field(unsigned int, pid) -+ __field(unsigned char, job_id) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ __entry->tgid = tgid; -+ __entry->pid = pid; -+ __entry->job_id = job_id; -+ ), -+ TP_printk("event=%u tgid=%u pid=%u job_id=%u", -+ __entry->event_id, __entry->tgid, __entry->pid, __entry->job_id) -+); -+ -+/** -+ * mali_pm_status - Called by mali_kbase_pm_driver.c -+ * @event_id: core type (shader, tiler, l2 cache) -+ * @value: 64bits bitmask reporting either power status of the cores (1-ON, 0-OFF) -+ */ -+TRACE_EVENT(mali_pm_status, -+ TP_PROTO(unsigned int event_id, unsigned long long value), -+ TP_ARGS(event_id, value), -+ TP_STRUCT__entry( -+ __field(unsigned int, event_id) -+ __field(unsigned long long, value) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ __entry->value = value; -+ ), -+ TP_printk("event %u = %llu", __entry->event_id, __entry->value) -+); -+ -+/** -+ * mali_pm_power_on - Called by mali_kbase_pm_driver.c -+ * @event_id: core type (shader, tiler, l2 cache) -+ * @value: 64bits bitmask reporting the cores to power up -+ */ -+TRACE_EVENT(mali_pm_power_on, -+ TP_PROTO(unsigned int event_id, unsigned long long value), -+ TP_ARGS(event_id, value), -+ TP_STRUCT__entry( -+ __field(unsigned int, event_id) -+ __field(unsigned long long, value) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ __entry->value = value; -+ ), -+ TP_printk("event %u = %llu", __entry->event_id, __entry->value) -+); -+ -+/** -+ * mali_pm_power_off - Called by mali_kbase_pm_driver.c -+ * @event_id: core type (shader, tiler, l2 cache) -+ * @value: 64bits bitmask reporting the cores to power down -+ */ -+TRACE_EVENT(mali_pm_power_off, -+ TP_PROTO(unsigned int event_id, unsigned long long value), -+ TP_ARGS(event_id, value), -+ TP_STRUCT__entry( -+ __field(unsigned int, event_id) -+ __field(unsigned long long, value) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ __entry->value = value; -+ ), -+ TP_printk("event %u = %llu", __entry->event_id, __entry->value) -+); -+ -+/** -+ * mali_page_fault_insert_pages - Called by page_fault_worker() -+ * it reports an MMU page fault resulting in new pages being mapped. -+ * @event_id: MMU address space number. -+ * @value: number of newly allocated pages -+ */ -+TRACE_EVENT(mali_page_fault_insert_pages, -+ TP_PROTO(int event_id, unsigned long value), -+ TP_ARGS(event_id, value), -+ TP_STRUCT__entry( -+ __field(int, event_id) -+ __field(unsigned long, value) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ __entry->value = value; -+ ), -+ TP_printk("event %d = %lu", __entry->event_id, __entry->value) -+); -+ -+/** -+ * mali_mmu_as_in_use - Called by assign_and_activate_kctx_addr_space() -+ * it reports that a certain MMU address space is in use now. -+ * @event_id: MMU address space number. -+ */ -+TRACE_EVENT(mali_mmu_as_in_use, -+ TP_PROTO(int event_id), -+ TP_ARGS(event_id), -+ TP_STRUCT__entry( -+ __field(int, event_id) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ ), -+ TP_printk("event=%d", __entry->event_id) -+); -+ -+/** -+ * mali_mmu_as_released - Called by kbasep_js_runpool_release_ctx_internal() -+ * it reports that a certain MMU address space has been released now. -+ * @event_id: MMU address space number. -+ */ -+TRACE_EVENT(mali_mmu_as_released, -+ TP_PROTO(int event_id), -+ TP_ARGS(event_id), -+ TP_STRUCT__entry( -+ __field(int, event_id) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ ), -+ TP_printk("event=%d", __entry->event_id) -+); -+ -+/** -+ * mali_total_alloc_pages_change - Called by kbase_atomic_add_pages() -+ * and by kbase_atomic_sub_pages() -+ * it reports that the total number of allocated pages is changed. -+ * @event_id: number of pages to be added or subtracted (according to the sign). -+ */ -+TRACE_EVENT(mali_total_alloc_pages_change, -+ TP_PROTO(long long int event_id), -+ TP_ARGS(event_id), -+ TP_STRUCT__entry( -+ __field(long long int, event_id) -+ ), -+ TP_fast_assign( -+ __entry->event_id = event_id; -+ ), -+ TP_printk("event=%lld", __entry->event_id) -+); -+ -+#endif /* _TRACE_MALI_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#undef linux -+#define TRACE_INCLUDE_PATH . -+ -+/* This part must be outside protection */ -+#include -diff --git a/drivers/gpu/arm/midgard/mali_malisw.h b/drivers/gpu/arm/midgard/mali_malisw.h -new file mode 100755 -index 000000000..99452933e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_malisw.h -@@ -0,0 +1,131 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * Kernel-wide include for common macros and types. -+ */ -+ -+#ifndef _MALISW_H_ -+#define _MALISW_H_ -+ -+#include -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) -+#define U8_MAX ((u8)~0U) -+#define S8_MAX ((s8)(U8_MAX>>1)) -+#define S8_MIN ((s8)(-S8_MAX - 1)) -+#define U16_MAX ((u16)~0U) -+#define S16_MAX ((s16)(U16_MAX>>1)) -+#define S16_MIN ((s16)(-S16_MAX - 1)) -+#define U32_MAX ((u32)~0U) -+#define S32_MAX ((s32)(U32_MAX>>1)) -+#define S32_MIN ((s32)(-S32_MAX - 1)) -+#define U64_MAX ((u64)~0ULL) -+#define S64_MAX ((s64)(U64_MAX>>1)) -+#define S64_MIN ((s64)(-S64_MAX - 1)) -+#endif /* LINUX_VERSION_CODE */ -+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) -+#define SIZE_MAX (~(size_t)0) -+#endif /* LINUX_VERSION_CODE */ -+ -+/** -+ * MIN - Return the lesser of two values. -+ * -+ * As a macro it may evaluate its arguments more than once. -+ * Refer to MAX macro for more details -+ */ -+#define MIN(x, y) ((x) < (y) ? (x) : (y)) -+ -+/** -+ * MAX - Return the greater of two values. -+ * -+ * As a macro it may evaluate its arguments more than once. -+ * If called on the same two arguments as MIN it is guaranteed to return -+ * the one that MIN didn't return. This is significant for types where not -+ * all values are comparable e.g. NaNs in floating-point types. But if you want -+ * to retrieve the min and max of two values, consider using a conditional swap -+ * instead. -+ */ -+#define MAX(x, y) ((x) < (y) ? (y) : (x)) -+ -+/** -+ * @hideinitializer -+ * Function-like macro for suppressing unused variable warnings. Where possible -+ * such variables should be removed; this macro is present for cases where we -+ * much support API backwards compatibility. -+ */ -+#define CSTD_UNUSED(x) ((void)(x)) -+ -+/** -+ * @hideinitializer -+ * Function-like macro for use where "no behavior" is desired. This is useful -+ * when compile time macros turn a function-like macro in to a no-op, but -+ * where having no statement is otherwise invalid. -+ */ -+#define CSTD_NOP(...) ((void)#__VA_ARGS__) -+ -+/** -+ * Function-like macro for converting a pointer in to a u64 for storing into -+ * an external data structure. This is commonly used when pairing a 32-bit -+ * CPU with a 64-bit peripheral, such as a Midgard GPU. C's type promotion -+ * is complex and a straight cast does not work reliably as pointers are -+ * often considered as signed. -+ */ -+#define PTR_TO_U64(x) ((uint64_t)((uintptr_t)(x))) -+ -+/** -+ * @hideinitializer -+ * Function-like macro for stringizing a single level macro. -+ * @code -+ * #define MY_MACRO 32 -+ * CSTD_STR1( MY_MACRO ) -+ * > "MY_MACRO" -+ * @endcode -+ */ -+#define CSTD_STR1(x) #x -+ -+/** -+ * @hideinitializer -+ * Function-like macro for stringizing a macro's value. This should not be used -+ * if the macro is defined in a way which may have no value; use the -+ * alternative @c CSTD_STR2N macro should be used instead. -+ * @code -+ * #define MY_MACRO 32 -+ * CSTD_STR2( MY_MACRO ) -+ * > "32" -+ * @endcode -+ */ -+#define CSTD_STR2(x) CSTD_STR1(x) -+ -+/** -+ * Specify an assertion value which is evaluated at compile time. Recommended -+ * usage is specification of a @c static @c INLINE function containing all of -+ * the assertions thus: -+ * -+ * @code -+ * static INLINE [module]_compile_time_assertions( void ) -+ * { -+ * COMPILE_TIME_ASSERT( sizeof(uintptr_t) == sizeof(intptr_t) ); -+ * } -+ * @endcode -+ * -+ * @note Use @c static not @c STATIC. We never want to turn off this @c static -+ * specification for testing purposes. -+ */ -+#define CSTD_COMPILE_TIME_ASSERT(expr) \ -+ do { switch (0) { case 0: case (expr):; } } while (false) -+ -+#endif /* _MALISW_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_midg_coherency.h b/drivers/gpu/arm/midgard/mali_midg_coherency.h -new file mode 100755 -index 000000000..a509cbd5f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_midg_coherency.h -@@ -0,0 +1,26 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _MIDG_COHERENCY_H_ -+#define _MIDG_COHERENCY_H_ -+ -+#define COHERENCY_ACE_LITE 0 -+#define COHERENCY_ACE 1 -+#define COHERENCY_NONE 31 -+#define COHERENCY_FEATURE_BIT(x) (1 << (x)) -+ -+#endif /* _MIDG_COHERENCY_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_midg_regmap.h b/drivers/gpu/arm/midgard/mali_midg_regmap.h -new file mode 100755 -index 000000000..7d7b7bcd3 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_midg_regmap.h -@@ -0,0 +1,611 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _MIDGARD_REGMAP_H_ -+#define _MIDGARD_REGMAP_H_ -+ -+#include "mali_midg_coherency.h" -+#include "mali_kbase_gpu_id.h" -+ -+/* -+ * Begin Register Offsets -+ */ -+ -+#define GPU_CONTROL_BASE 0x0000 -+#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) -+#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ -+#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ -+#define SUSPEND_SIZE 0x008 /* (RO) Fixed-function suspend buffer -+ size */ -+#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ -+#define MEM_FEATURES 0x010 /* (RO) Memory system features */ -+#define MMU_FEATURES 0x014 /* (RO) MMU features */ -+#define AS_PRESENT 0x018 /* (RO) Address space slots present */ -+#define JS_PRESENT 0x01C /* (RO) Job slots present */ -+#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ -+#define GPU_IRQ_CLEAR 0x024 /* (WO) */ -+#define GPU_IRQ_MASK 0x028 /* (RW) */ -+#define GPU_IRQ_STATUS 0x02C /* (RO) */ -+ -+/* IRQ flags */ -+#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ -+#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ -+#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. Intended to use with SOFT_RESET -+ commands which may take time. */ -+#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ -+#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down -+ and the power manager is idle. */ -+ -+#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ -+#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ -+ -+#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ -+ | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) -+ -+#define GPU_COMMAND 0x030 /* (WO) */ -+#define GPU_STATUS 0x034 /* (RO) */ -+#define LATEST_FLUSH 0x038 /* (RO) */ -+ -+#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ -+#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ -+ -+#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ -+#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ -+#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ -+ -+#define PWR_KEY 0x050 /* (WO) Power manager key register */ -+#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ -+#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ -+ -+#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ -+#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ -+#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ -+#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ -+#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ -+#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ -+#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ -+ -+#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ -+#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ -+#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ -+#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ -+ -+#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ -+#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ -+#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ -+#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ -+ -+#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ -+#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ -+#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ -+ -+#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) -+ -+#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ -+#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ -+#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ -+#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ -+#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ -+#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ -+#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ -+#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ -+#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ -+#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ -+#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ -+#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ -+#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ -+#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ -+#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ -+#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ -+ -+#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) -+ -+#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ -+#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ -+ -+#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ -+#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ -+ -+#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ -+#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ -+ -+#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ -+#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ -+ -+ -+#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ -+#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ -+ -+#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ -+#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ -+ -+#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ -+#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ -+ -+#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ -+#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ -+ -+ -+#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ -+#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ -+ -+#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ -+#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ -+ -+#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ -+#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ -+ -+#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ -+#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ -+ -+ -+#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ -+#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ -+ -+#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ -+#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ -+ -+#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ -+#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ -+ -+#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ -+#define STACK_PRWOFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ -+ -+ -+#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ -+#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ -+ -+#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ -+#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ -+ -+#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ -+#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ -+ -+#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ -+#define STACK_PRWTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ -+ -+ -+#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ -+#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ -+ -+#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ -+#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ -+ -+#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ -+#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ -+ -+#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ -+#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ -+ -+#define JM_CONFIG 0xF00 /* (RW) Job Manager configuration register (Implementation specific register) */ -+#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */ -+#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */ -+#define L2_MMU_CONFIG 0xF0C /* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */ -+ -+#define JOB_CONTROL_BASE 0x1000 -+ -+#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) -+ -+#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ -+#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ -+#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ -+#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ -+#define JOB_IRQ_JS_STATE 0x010 /* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ -+#define JOB_IRQ_THROTTLE 0x014 /* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS is NOT affected by this, just the delivery of the interrupt. */ -+ -+#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ -+#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ -+#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ -+#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ -+#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ -+#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ -+#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ -+#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ -+#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ -+#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ -+#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ -+#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ -+#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ -+#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ -+#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ -+#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ -+ -+#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) -+ -+#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ -+#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ -+#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ -+#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ -+#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ -+#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ -+#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ -+#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job -+ slot n */ -+ -+#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ -+#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ -+ -+#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ -+#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ -+ -+#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ -+#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ -+#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ -+#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for -+ job slot n */ -+ -+#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ -+ -+#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ -+ -+#define MEMORY_MANAGEMENT_BASE 0x2000 -+#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) -+ -+#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ -+#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ -+#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ -+#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ -+ -+#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ -+#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ -+#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ -+#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ -+#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ -+#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ -+#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ -+#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ -+#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ -+#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ -+#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ -+#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ -+#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ -+#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ -+#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ -+#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ -+ -+#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) -+ -+#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ -+#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ -+#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ -+#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ -+#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ -+#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ -+#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ -+#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ -+#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ -+#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ -+#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ -+ -+ -+/* (RW) Translation table configuration for address space n, low word */ -+#define AS_TRANSCFG_LO 0x30 -+/* (RW) Translation table configuration for address space n, high word */ -+#define AS_TRANSCFG_HI 0x34 -+/* (RO) Secondary fault address for address space n, low word */ -+#define AS_FAULTEXTRA_LO 0x38 -+/* (RO) Secondary fault address for address space n, high word */ -+#define AS_FAULTEXTRA_HI 0x3C -+ -+/* End Register Offsets */ -+ -+/* -+ * MMU_IRQ_RAWSTAT register values. Values are valid also for -+ MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. -+ */ -+ -+#define MMU_PAGE_FAULT_FLAGS 16 -+ -+/* Macros returning a bitmask to retrieve page fault or bus error flags from -+ * MMU registers */ -+#define MMU_PAGE_FAULT(n) (1UL << (n)) -+#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) -+ -+/* -+ * Begin LPAE MMU TRANSTAB register values -+ */ -+#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 -+#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) -+#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) -+#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) -+#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) -+#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) -+ -+#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 -+ -+/* -+ * Begin AARCH64 MMU TRANSTAB register values -+ */ -+#define MMU_HW_OUTA_BITS 40 -+#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) -+ -+/* -+ * Begin MMU STATUS register values -+ */ -+#define AS_STATUS_AS_ACTIVE 0x01 -+ -+#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) -+ -+#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) -+ -+#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8) -+#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0<<8) -+#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8) -+#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8) -+#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8) -+ -+/* -+ * Begin MMU TRANSCFG register values -+ */ -+ -+#define AS_TRANSCFG_ADRMODE_LEGACY 0 -+#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 -+#define AS_TRANSCFG_ADRMODE_IDENTITY 2 -+#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 -+#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 -+ -+#define AS_TRANSCFG_ADRMODE_MASK 0xF -+ -+ -+/* -+ * Begin TRANSCFG register values -+ */ -+#define AS_TRANSCFG_PTW_MEMATTR_MASK (3 << 24) -+#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1 << 24) -+#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2 << 24) -+ -+#define AS_TRANSCFG_PTW_SH_MASK ((3 << 28)) -+#define AS_TRANSCFG_PTW_SH_OS (2 << 28) -+#define AS_TRANSCFG_PTW_SH_IS (3 << 28) -+ -+/* -+ * Begin Command Values -+ */ -+ -+/* JS_COMMAND register commands */ -+#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ -+#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ -+#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ -+#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ -+#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ -+#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ -+#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ -+#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ -+ -+#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ -+ -+/* AS_COMMAND register commands */ -+#define AS_COMMAND_NOP 0x00 /* NOP Operation */ -+#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ -+#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ -+#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ -+#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs -+ (deprecated - only for use with T60x) */ -+#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ -+#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then -+ flush all L2 caches then issue a flush region command to all MMUs */ -+ -+/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ -+#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) -+#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) -+#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) -+#define JS_CONFIG_START_MMU (1u << 10) -+#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) -+#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION -+#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) -+#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) -+#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) -+#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) -+#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) -+ -+/* JS_XAFFINITY register values */ -+#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) -+#define JS_XAFFINITY_TILER_ENABLE (1u << 8) -+#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) -+ -+/* JS_STATUS register values */ -+ -+/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. -+ * The values are separated to avoid dependency of userspace and kernel code. -+ */ -+ -+/* Group of values representing the job status insead a particular fault */ -+#define JS_STATUS_NO_EXCEPTION_BASE 0x00 -+#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ -+#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ -+#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ -+ -+/* General fault values */ -+#define JS_STATUS_FAULT_BASE 0x40 -+#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ -+#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ -+#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ -+#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ -+#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ -+#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ -+ -+/* Instruction or data faults */ -+#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 -+#define JS_STATUS_INSTR_INVALID_PC (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ -+#define JS_STATUS_INSTR_INVALID_ENC (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ -+#define JS_STATUS_INSTR_TYPE_MISMATCH (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ -+#define JS_STATUS_INSTR_OPERAND_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ -+#define JS_STATUS_INSTR_TLS_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ -+#define JS_STATUS_INSTR_BARRIER_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ -+#define JS_STATUS_INSTR_ALIGN_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ -+/* NOTE: No fault with 0x57 code defined in spec. */ -+#define JS_STATUS_DATA_INVALID_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ -+#define JS_STATUS_TILE_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ -+#define JS_STATUS_ADDRESS_RANGE_FAULT (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ -+ -+/* Other faults */ -+#define JS_STATUS_MEMORY_FAULT_BASE 0x60 -+#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ -+#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ -+ -+/* GPU_COMMAND values */ -+#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ -+#define GPU_COMMAND_SOFT_RESET 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ -+#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ -+#define GPU_COMMAND_PRFCNT_CLEAR 0x03 /* Clear all performance counters, setting them all to zero. */ -+#define GPU_COMMAND_PRFCNT_SAMPLE 0x04 /* Sample all performance counters, writing them out to memory */ -+#define GPU_COMMAND_CYCLE_COUNT_START 0x05 /* Starts the cycle counter, and system timestamp propagation */ -+#define GPU_COMMAND_CYCLE_COUNT_STOP 0x06 /* Stops the cycle counter, and system timestamp propagation */ -+#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ -+#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ -+#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ -+ -+/* End Command Values */ -+ -+/* GPU_STATUS values */ -+#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ -+#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ -+ -+/* PRFCNT_CONFIG register values */ -+#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ -+#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ -+#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ -+ -+#define PRFCNT_CONFIG_MODE_OFF 0 /* The performance counters are disabled. */ -+#define PRFCNT_CONFIG_MODE_MANUAL 1 /* The performance counters are enabled, but are only written out when a PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. */ -+#define PRFCNT_CONFIG_MODE_TILE 2 /* The performance counters are enabled, and are written out each time a tile finishes rendering. */ -+ -+/* AS_MEMATTR values: */ -+/* Use GPU implementation-defined caching policy. */ -+#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull -+/* The attribute set to force all resources to be cached. */ -+#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full -+/* Inner write-alloc cache setup, no outer caching */ -+#define AS_MEMATTR_WRITE_ALLOC 0x8Dull -+ -+/* Set to implementation defined, outer caching */ -+#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull -+/* Set to write back memory, outer caching */ -+#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull -+ -+/* Use GPU implementation-defined caching policy. */ -+#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull -+/* The attribute set to force all resources to be cached. */ -+#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full -+/* Inner write-alloc cache setup, no outer caching */ -+#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull -+/* Set to implementation defined, outer caching */ -+#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull -+/* Set to write back memory, outer caching */ -+#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull -+ -+/* Symbol for default MEMATTR to use */ -+ -+/* Default is - HW implementation defined caching */ -+#define AS_MEMATTR_INDEX_DEFAULT 0 -+#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 -+ -+/* HW implementation defined caching */ -+#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 -+/* Force cache on */ -+#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 -+/* Write-alloc */ -+#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 -+/* Outer coherent, inner implementation defined policy */ -+#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 -+/* Outer coherent, write alloc inner */ -+#define AS_MEMATTR_INDEX_OUTER_WA 4 -+ -+/* JS_FEATURES register */ -+ -+#define JS_FEATURE_NULL_JOB (1u << 1) -+#define JS_FEATURE_SET_VALUE_JOB (1u << 2) -+#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) -+#define JS_FEATURE_COMPUTE_JOB (1u << 4) -+#define JS_FEATURE_VERTEX_JOB (1u << 5) -+#define JS_FEATURE_GEOMETRY_JOB (1u << 6) -+#define JS_FEATURE_TILER_JOB (1u << 7) -+#define JS_FEATURE_FUSED_JOB (1u << 8) -+#define JS_FEATURE_FRAGMENT_JOB (1u << 9) -+ -+/* End JS_FEATURES register */ -+ -+/* L2_MMU_CONFIG register */ -+#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) -+#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT (24) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT) -+ -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT (26) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) -+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT) -+/* End L2_MMU_CONFIG register */ -+ -+/* THREAD_* registers */ -+ -+/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ -+#define IMPLEMENTATION_UNSPECIFIED 0 -+#define IMPLEMENTATION_SILICON 1 -+#define IMPLEMENTATION_FPGA 2 -+#define IMPLEMENTATION_MODEL 3 -+ -+/* Default values when registers are not supported by the implemented hardware */ -+#define THREAD_MT_DEFAULT 256 -+#define THREAD_MWS_DEFAULT 256 -+#define THREAD_MBS_DEFAULT 256 -+#define THREAD_MR_DEFAULT 1024 -+#define THREAD_MTQ_DEFAULT 4 -+#define THREAD_MTGS_DEFAULT 10 -+ -+/* End THREAD_* registers */ -+ -+/* SHADER_CONFIG register */ -+ -+#define SC_ALT_COUNTERS (1ul << 3) -+#define SC_OVERRIDE_FWD_PIXEL_KILL (1ul << 4) -+#define SC_SDC_DISABLE_OQ_DISCARD (1ul << 6) -+#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) -+#define SC_LS_PAUSEBUFFER_DISABLE (1ul << 16) -+#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) -+#define SC_ENABLE_TEXGRD_FLAGS (1ul << 25) -+/* End SHADER_CONFIG register */ -+ -+/* TILER_CONFIG register */ -+ -+#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) -+ -+/* End TILER_CONFIG register */ -+ -+/* JM_CONFIG register */ -+ -+#define JM_TIMESTAMP_OVERRIDE (1ul << 0) -+#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) -+#define JM_JOB_THROTTLE_ENABLE (1ul << 2) -+#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) -+#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) -+#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) -+#define JM_IDVS_GROUP_SIZE_SHIFT (16) -+#define JM_MAX_IDVS_GROUP_SIZE (0x3F) -+/* End JM_CONFIG register */ -+ -+ -+#endif /* _MIDGARD_REGMAP_H_ */ -diff --git a/drivers/gpu/arm/midgard/mali_timeline.h b/drivers/gpu/arm/midgard/mali_timeline.h -new file mode 100755 -index 000000000..bd5f6614b ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_timeline.h -@@ -0,0 +1,396 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM mali_timeline -+ -+#if !defined(_MALI_TIMELINE_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _MALI_TIMELINE_H -+ -+#include -+ -+TRACE_EVENT(mali_timeline_atoms_in_flight, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int tgid, -+ int count), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ tgid, -+ count), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, tgid) -+ __field(int, count) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->tgid = tgid; -+ __entry->count = count; -+ ), -+ -+ TP_printk("%i,%i.%.9i,%i,%i", CTX_SET_NR_ATOMS_IN_FLIGHT, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->tgid, -+ __entry->count) -+); -+ -+ -+TRACE_EVENT(mali_timeline_atom, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int tgid, -+ int atom_id), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ tgid, -+ atom_id), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, tgid) -+ __field(int, atom_id) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->tgid = tgid; -+ __entry->atom_id = atom_id; -+ ), -+ -+ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->tgid, -+ __entry->atom_id, -+ __entry->atom_id) -+); -+ -+TRACE_EVENT(mali_timeline_gpu_slot_active, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int tgid, -+ int js, -+ int count), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ tgid, -+ js, -+ count), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, tgid) -+ __field(int, js) -+ __field(int, count) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->tgid = tgid; -+ __entry->js = js; -+ __entry->count = count; -+ ), -+ -+ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->tgid, -+ __entry->js, -+ __entry->count) -+); -+ -+TRACE_EVENT(mali_timeline_gpu_slot_action, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int tgid, -+ int js, -+ int count), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ tgid, -+ js, -+ count), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, tgid) -+ __field(int, js) -+ __field(int, count) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->tgid = tgid; -+ __entry->js = js; -+ __entry->count = count; -+ ), -+ -+ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->tgid, -+ __entry->js, -+ __entry->count) -+); -+ -+TRACE_EVENT(mali_timeline_gpu_power_active, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int active), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ active), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, active) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->active = active; -+ ), -+ -+ TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->active) -+ -+); -+ -+TRACE_EVENT(mali_timeline_l2_power_active, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int state), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ state), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, state) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->state = state; -+ ), -+ -+ TP_printk("%i,%i.%.9i,0,%i", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->state) -+ -+); -+TRACE_EVENT(mali_timeline_pm_event, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int pm_event_type, -+ unsigned int pm_event_id), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ pm_event_type, -+ pm_event_id), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, pm_event_type) -+ __field(unsigned int, pm_event_id) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->pm_event_type = pm_event_type; -+ __entry->pm_event_id = pm_event_id; -+ ), -+ -+ TP_printk("%i,%i.%.9i,0,%i,%u", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->pm_event_type, __entry->pm_event_id) -+ -+); -+ -+TRACE_EVENT(mali_timeline_slot_atom, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int event_type, -+ int tgid, -+ int js, -+ int atom_id), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ event_type, -+ tgid, -+ js, -+ atom_id), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, event_type) -+ __field(int, tgid) -+ __field(int, js) -+ __field(int, atom_id) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->event_type = event_type; -+ __entry->tgid = tgid; -+ __entry->js = js; -+ __entry->atom_id = atom_id; -+ ), -+ -+ TP_printk("%i,%i.%.9i,%i,%i,%i", __entry->event_type, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->tgid, -+ __entry->js, -+ __entry->atom_id) -+); -+ -+TRACE_EVENT(mali_timeline_pm_checktrans, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int trans_code, -+ int trans_id), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ trans_code, -+ trans_id), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, trans_code) -+ __field(int, trans_id) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->trans_code = trans_code; -+ __entry->trans_id = trans_id; -+ ), -+ -+ TP_printk("%i,%i.%.9i,0,%i", __entry->trans_code, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->trans_id) -+ -+); -+ -+TRACE_EVENT(mali_timeline_context_active, -+ -+ TP_PROTO(u64 ts_sec, -+ u32 ts_nsec, -+ int count), -+ -+ TP_ARGS(ts_sec, -+ ts_nsec, -+ count), -+ -+ TP_STRUCT__entry( -+ __field(u64, ts_sec) -+ __field(u32, ts_nsec) -+ __field(int, count) -+ ), -+ -+ TP_fast_assign( -+ __entry->ts_sec = ts_sec; -+ __entry->ts_nsec = ts_nsec; -+ __entry->count = count; -+ ), -+ -+ TP_printk("%i,%i.%.9i,0,%i", SW_SET_CONTEXT_ACTIVE, -+ (int)__entry->ts_sec, -+ (int)__entry->ts_nsec, -+ __entry->count) -+); -+ -+#endif /* _MALI_TIMELINE_H */ -+ -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH . -+ -+/* This part must be outside protection */ -+#include -+ -diff --git a/drivers/gpu/arm/midgard/mali_uk.h b/drivers/gpu/arm/midgard/mali_uk.h -new file mode 100755 -index 000000000..841d03fb5 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/mali_uk.h -@@ -0,0 +1,141 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010, 2012-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+/** -+ * @file mali_uk.h -+ * Types and definitions that are common across OSs for both the user -+ * and kernel side of the User-Kernel interface. -+ */ -+ -+#ifndef _UK_H_ -+#define _UK_H_ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif /* __cplusplus */ -+ -+/** -+ * @addtogroup base_api -+ * @{ -+ */ -+ -+/** -+ * @defgroup uk_api User-Kernel Interface API -+ * -+ * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device -+ * drivers developed as part of the Midgard DDK. Currently that includes the Base driver and the UMP driver. -+ * -+ * It exposes an OS independent API to user-side code (UKU) which routes functions calls to an OS-independent -+ * kernel-side API (UKK) via an OS-specific communication mechanism. -+ * -+ * This API is internal to the Midgard DDK and is not exposed to any applications. -+ * -+ * @{ -+ */ -+ -+/** -+ * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The -+ * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this -+ * identifier to select a UKK client to the uku_open() function. -+ * -+ * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id -+ * enumeration and the uku_open() implemenation for the various OS ports need to be updated to -+ * provide a mapping of the identifier to the OS specific device name. -+ * -+ */ -+enum uk_client_id { -+ /** -+ * Value used to identify the Base driver UK client. -+ */ -+ UK_CLIENT_MALI_T600_BASE, -+ -+ /** The number of uk clients supported. This must be the last member of the enum */ -+ UK_CLIENT_COUNT -+}; -+ -+/** -+ * Each function callable through the UK interface has a unique number. -+ * Functions provided by UK clients start from number UK_FUNC_ID. -+ * Numbers below UK_FUNC_ID are used for internal UK functions. -+ */ -+enum uk_func { -+ UKP_FUNC_ID_CHECK_VERSION, /**< UKK Core internal function */ -+ /** -+ * Each UK client numbers the functions they provide starting from -+ * number UK_FUNC_ID. This number is then eventually assigned to the -+ * id field of the union uk_header structure when preparing to make a -+ * UK call. See your UK client for a list of their function numbers. -+ */ -+ UK_FUNC_ID = 512 -+}; -+ -+/** -+ * Arguments for a UK call are stored in a structure. This structure consists -+ * of a fixed size header and a payload. The header carries a 32-bit number -+ * identifying the UK function to be called (see uk_func). When the UKK client -+ * receives this header and executed the requested UK function, it will use -+ * the same header to store the result of the function in the form of a -+ * int return code. The size of this structure is such that the -+ * first member of the payload following the header can be accessed efficiently -+ * on a 32 and 64-bit kernel and the structure has the same size regardless -+ * of a 32 or 64-bit kernel. The uk_kernel_size_type type should be defined -+ * accordingly in the OS specific mali_uk_os.h header file. -+ */ -+union uk_header { -+ /** -+ * 32-bit number identifying the UK function to be called. -+ * Also see uk_func. -+ */ -+ u32 id; -+ /** -+ * The int return code returned by the called UK function. -+ * See the specification of the particular UK function you are -+ * calling for the meaning of the error codes returned. All -+ * UK functions return 0 on success. -+ */ -+ u32 ret; -+ /* -+ * Used to ensure 64-bit alignment of this union. Do not remove. -+ * This field is used for padding and does not need to be initialized. -+ */ -+ u64 sizer; -+}; -+ -+/** -+ * This structure carries a 16-bit major and minor number and is sent along with an internal UK call -+ * used during uku_open to identify the versions of the UK module in use by the user-side and kernel-side. -+ */ -+struct uku_version_check_args { -+ union uk_header header; -+ /**< UK call header */ -+ u16 major; -+ /**< This field carries the user-side major version on input and the kernel-side major version on output */ -+ u16 minor; -+ /**< This field carries the user-side minor version on input and the kernel-side minor version on output. */ -+ u8 padding[4]; -+}; -+ -+/** @} end group uk_api */ -+ -+/** @} *//* end group base_api */ -+ -+#ifdef __cplusplus -+} -+#endif /* __cplusplus */ -+#endif /* _UK_H_ */ -diff --git a/drivers/gpu/arm/midgard/platform/Kconfig b/drivers/gpu/arm/midgard/platform/Kconfig -new file mode 100755 -index 000000000..8fb4e917c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/Kconfig -@@ -0,0 +1,24 @@ -+# -+# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ -+ -+# Add your platform specific Kconfig file here -+# -+# "drivers/gpu/arm/midgard/platform/xxx/Kconfig" -+# -+# Where xxx is the platform name is the name set in MALI_PLATFORM_THIRDPARTY_NAME -+# -+ -diff --git a/drivers/gpu/arm/midgard/platform/devicetree/Kbuild b/drivers/gpu/arm/midgard/platform/devicetree/Kbuild -new file mode 100755 -index 000000000..e888a42fc ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/devicetree/Kbuild -@@ -0,0 +1,18 @@ -+# -+# (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+mali_kbase-y += \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_devicetree.o \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_runtime_pm.o -diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c -new file mode 100755 -index 000000000..b2a7c93f1 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c -@@ -0,0 +1,31 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+ -+int kbase_platform_early_init(void) -+{ -+ /* Nothing needed at this stage */ -+ return 0; -+} -+ -+static struct kbase_platform_config dummy_platform_config; -+ -+struct kbase_platform_config *kbase_get_platform_config(void) -+{ -+ return &dummy_platform_config; -+} -diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -new file mode 100755 -index 000000000..49e107f98 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -@@ -0,0 +1,73 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * Maximum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MAX (5000) -+/** -+ * Minimum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MIN (5000) -+ -+/** -+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock -+ * -+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_cpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define CPU_SPEED_FUNC (NULL) -+ -+/** -+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock -+ * -+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_gpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define GPU_SPEED_FUNC (NULL) -+ -+/** -+ * Power management configuration -+ * -+ * Attached value: pointer to @ref kbase_pm_callback_conf -+ * Default value: See @ref kbase_pm_callback_conf -+ */ -+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) -+ -+/** -+ * Platform specific configuration functions -+ * -+ * Attached value: pointer to @ref kbase_platform_funcs_conf -+ * Default value: See @ref kbase_platform_funcs_conf -+ */ -+#define PLATFORM_FUNCS (NULL) -+ -+extern struct kbase_pm_callback_conf pm_callbacks; -diff --git a/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c -new file mode 100755 -index 000000000..aa4376afd ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c -@@ -0,0 +1,100 @@ -+/* -+ * -+ * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+ -+static int pm_callback_power_on(struct kbase_device *kbdev) -+{ -+ int ret; -+ -+ dev_dbg(kbdev->dev, "pm_callback_power_on %p\n", -+ (void *)kbdev->dev->pm_domain); -+ -+ ret = pm_runtime_get_sync(kbdev->dev); -+ -+ dev_dbg(kbdev->dev, "pm_runtime_get returned %d\n", ret); -+ -+ return 1; -+} -+ -+static void pm_callback_power_off(struct kbase_device *kbdev) -+{ -+ dev_dbg(kbdev->dev, "pm_callback_power_off\n"); -+ -+ pm_runtime_put_autosuspend(kbdev->dev); -+} -+ -+int kbase_device_runtime_init(struct kbase_device *kbdev) -+{ -+ dev_dbg(kbdev->dev, "kbase_device_runtime_init\n"); -+ pm_runtime_enable(kbdev->dev); -+ -+ return 0; -+} -+ -+void kbase_device_runtime_disable(struct kbase_device *kbdev) -+{ -+ dev_dbg(kbdev->dev, "kbase_device_runtime_disable\n"); -+ pm_runtime_disable(kbdev->dev); -+} -+ -+static int pm_callback_runtime_on(struct kbase_device *kbdev) -+{ -+ dev_dbg(kbdev->dev, "pm_callback_runtime_on\n"); -+ -+ return 0; -+} -+ -+static void pm_callback_runtime_off(struct kbase_device *kbdev) -+{ -+ dev_dbg(kbdev->dev, "pm_callback_runtime_off\n"); -+} -+ -+static void pm_callback_resume(struct kbase_device *kbdev) -+{ -+ int ret = pm_callback_runtime_on(kbdev); -+ -+ WARN_ON(ret); -+} -+ -+static void pm_callback_suspend(struct kbase_device *kbdev) -+{ -+ pm_callback_runtime_off(kbdev); -+} -+ -+struct kbase_pm_callback_conf pm_callbacks = { -+ .power_on_callback = pm_callback_power_on, -+ .power_off_callback = pm_callback_power_off, -+ .power_suspend_callback = pm_callback_suspend, -+ .power_resume_callback = pm_callback_resume, -+#ifdef KBASE_PM_RUNTIME -+ .power_runtime_init_callback = kbase_device_runtime_init, -+ .power_runtime_term_callback = kbase_device_runtime_disable, -+ .power_runtime_on_callback = pm_callback_runtime_on, -+ .power_runtime_off_callback = pm_callback_runtime_off, -+#else /* KBASE_PM_RUNTIME */ -+ .power_runtime_init_callback = NULL, -+ .power_runtime_term_callback = NULL, -+ .power_runtime_on_callback = NULL, -+ .power_runtime_off_callback = NULL, -+#endif /* KBASE_PM_RUNTIME */ -+}; -+ -+ -diff --git a/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h -new file mode 100755 -index 000000000..c11085af5 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_common.h -@@ -0,0 +1,28 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2013 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+#include -+ -+ -+/** -+ * @brief Entry point to transfer control to a platform for early initialization -+ * -+ * This function is called early on in the initialization during execution of -+ * @ref kbase_driver_init. -+ * -+ * @return Zero to indicate success non-zero for failure. -+ */ -+int kbase_platform_early_init(void); -+int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev); -diff --git a/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h -new file mode 100755 -index 000000000..01f9dfce9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/mali_kbase_platform_fake.h -@@ -0,0 +1,38 @@ -+/* -+ * -+ * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifdef CONFIG_MALI_PLATFORM_FAKE -+ -+/** -+ * kbase_platform_fake_register - Entry point for fake platform registration -+ * -+ * This function is called early on in the initialization during execution of -+ * kbase_driver_init. -+ * -+ * Return: 0 to indicate success, non-zero for failure. -+ */ -+int kbase_platform_fake_register(void); -+ -+/** -+ * kbase_platform_fake_unregister - Entry point for fake platform unregistration -+ * -+ * This function is called in the termination during execution of -+ * kbase_driver_exit. -+ */ -+void kbase_platform_fake_unregister(void); -+ -+#endif /* CONFIG_MALI_PLATFORM_FAKE */ -diff --git a/drivers/gpu/arm/midgard/platform/rk/Kbuild b/drivers/gpu/arm/midgard/platform/rk/Kbuild -new file mode 100755 -index 000000000..db993487e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/rk/Kbuild -@@ -0,0 +1,17 @@ -+# -+# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+midgard_kbase-y += \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_rk.o -+ -diff --git a/drivers/gpu/arm/midgard/platform/rk/custom_log.h b/drivers/gpu/arm/midgard/platform/rk/custom_log.h -new file mode 100755 -index 000000000..fe5e12241 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/rk/custom_log.h -@@ -0,0 +1,209 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* ---------------------------------------------------------------------------- -+ * File: custom_log.h -+ * -+ * Desc: ChenZhen å好的 log 输出的定制实现. -+ * -+ * -------------------------------------------------------------------- -+ * < 习语 å’Œ 缩略语 > : -+ * -+ * -------------------------------------------------------------------- -+ * Usage: -+ * -+ * Note: -+ * -+ * Author: ChenZhen -+ * -+ * ---------------------------------------------------------------------------- -+ * Version: -+ * v1.0 -+ * ---------------------------------------------------------------------------- -+ * Log: -+ ----Fri Nov 19 15:20:28 2010 v1.0 -+ * -+ * ---------------------------------------------------------------------------- -+ */ -+ -+#ifndef __CUSTOM_LOG_H__ -+#define __CUSTOM_LOG_H__ -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/* ----------------------------------------------------------------------------- -+ * Include Files -+ * ----------------------------------------------------------------------------- -+ */ -+#include -+#include -+ -+/* ----------------------------------------------------------------------------- -+ * Macros Definition -+ * ----------------------------------------------------------------------------- -+ */ -+ -+/** 若下列 macro 有被定义, æ‰ ä½¿èƒ½ log 输出. */ -+/* #define ENABLE_DEBUG_LOG */ -+ -+/*----------------------------------------------------------------------------*/ -+ -+#ifdef ENABLE_VERBOSE_LOG -+/** Verbose log. */ -+#define V(fmt, args...) \ -+ pr_debug("V : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+#else -+#define V(...) ((void)0) -+#endif -+ -+#ifdef ENABLE_DEBUG_LOG -+/** Debug log. */ -+#define D(fmt, args...) \ -+ pr_info("D : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+#else -+#define D(...) ((void)0) -+#endif -+ -+#define I(fmt, args...) \ -+ pr_info("I : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+ -+#define W(fmt, args...) \ -+ pr_warn("W : [File] : %s; [Line] : %d; [Func] : %s(); " \ -+ fmt "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+ -+#define E(fmt, args...) \ -+ pr_err("E : [File] : %s; [Line] : %d; [Func] : %s(); " fmt \ -+ "\n", \ -+ __FILE__, \ -+ __LINE__, \ -+ __func__, \ -+ ## args) -+ -+/*-------------------------------------------------------*/ -+ -+/** 使用 D(), 以å进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ -+#define D_DEC(var) D(#var " = %d.", var) -+ -+#define E_DEC(var) E(#var " = %d.", var) -+ -+/** 使用 D(), 以å六进制的形å¼æ‰“å°å˜é‡ 'var' çš„ value. */ -+#define D_HEX(var) D(#var " = 0x%x.", var) -+ -+#define E_HEX(var) E(#var " = 0x%x.", var) -+ -+/** -+ * 使用 D(), 以å六进制的形å¼, -+ * 打å°æŒ‡é’ˆç±»åž‹å˜é‡ 'ptr' çš„ value. -+ */ -+#define D_PTR(ptr) D(#ptr " = %p.", ptr) -+ -+#define E_PTR(ptr) E(#ptr " = %p.", ptr) -+ -+/** 使用 D(), æ‰“å° char 字串. */ -+#define D_STR(p_str) \ -+do { \ -+ if (!p_str) { \ -+ D(#p_str " = NULL."); \ -+ else \ -+ D(#p_str " = '%s'.", p_str); \ -+} while (0) -+ -+#define E_STR(p_str) \ -+do { \ -+ if (!p_str) \ -+ E(#p_str " = NULL."); \ -+ else \ -+ E(#p_str " = '%s'.", p_str); \ -+} while (0) -+ -+#ifdef ENABLE_DEBUG_LOG -+/** -+ * log 从 'p_start' 地å€å¼€å§‹çš„ 'len' 个字节的数æ®. -+ */ -+#define D_MEM(p_start, len) \ -+do { \ -+ int i = 0; \ -+ char *p = (char *)(p_start); \ -+ D("dump memory from addr of '" #p_start "', from %p, length %d' : ", \ -+ (p_start), \ -+ (len)); \ -+ pr_debug("\t\t"); \ -+ for (i = 0; i < (len); i++) \ -+ pr_debug("0x%02x, ", p[i]); \ -+ pr_debug("\n"); \ -+} while (0) -+#else -+#define D_MEM(...) ((void)0) -+#endif -+ -+/*-------------------------------------------------------*/ -+ -+/** -+ * 在特定æ¡ä»¶ä¸‹, 判定 error å‘生, -+ * å°†å˜é‡ 'ret_var' 设置 'err_code', -+ * log 输出对应的 Error Caution, -+ * 然åŽè·³è½¬ 'label' 指定的代ç å¤„执行. -+ * @param msg -+ * 纯字串形å¼çš„æ示信æ¯. -+ * @param ret_var -+ * 标识函数执行状æ€æˆ–者结果的å˜é‡, -+ * 将被设置具体的 Error Code. -+ * 通常是 'ret' or 'result'. -+ * @param err_code -+ * 表å¾ç‰¹å®š error 的常数标识, -+ * 通常是 å®çš„å½¢æ€. -+ * @param label -+ * 程åºå°†è¦è·³è½¬åˆ°çš„错误处ç†ä»£ç çš„æ ‡å·, -+ * 通常就是 'EXIT'. -+ * @param args... -+ * 对应 'msg_fmt' 实å‚中, -+ * '%s', '%d', ... 等转æ¢è¯´æ˜Žç¬¦çš„具体å¯å˜é•¿å®žå‚. -+ */ -+#define SET_ERROR_AND_JUMP(msg_fmt, ret_var, err_code, label, args...) \ -+do { \ -+ E("To set '" #ret_var "' to %d('" #err_code "'), because : " msg_fmt, \ -+ (err_code), \ -+ ## args); \ -+ (ret_var) = (err_code); \ -+ goto label; \ -+} while (0) -+ -+/* ----------------------------------------------------------------------------- -+ * Types and Structures Definition -+ * ----------------------------------------------------------------------------- -+ */ -+ -+/* ----------------------------------------------------------------------------- -+ * Global Functions' Prototype -+ * ----------------------------------------------------------------------------- -+ */ -+ -+/* ----------------------------------------------------------------------------- -+ * Inline Functions Implementation -+ * ----------------------------------------------------------------------------- -+ */ -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* __CUSTOM_LOG_H__ */ -diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h -new file mode 100755 -index 000000000..07c5b6f8a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_platform.h -@@ -0,0 +1,88 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ */ -+ -+/** -+ * @file mali_kbase_config_platform.h -+ * 声明 platform_config_of_rk (platform_rk çš„ platform_config). -+ */ -+ -+/** -+ * Maximum frequency GPU will be clocked at. -+ * Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MAX (5000) -+ -+/** -+ * Minimum frequency GPU will be clocked at. -+ * Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MIN (5000) -+ -+/** -+ * CPU_SPEED_FUNC -+ * - A pointer to a function that calculates the CPU clock -+ * -+ * CPU clock speed of the platform is in MHz -+ * - see kbase_cpu_clk_speed_func for the function prototype. -+ * -+ * Attached value: A kbase_cpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define CPU_SPEED_FUNC (NULL) -+ -+/** -+ * GPU_SPEED_FUNC -+ * - A pointer to a function that calculates the GPU clock -+ * -+ * GPU clock speed of the platform in MHz -+ * - see kbase_gpu_clk_speed_func for the function prototype. -+ * -+ * Attached value: A kbase_gpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define GPU_SPEED_FUNC (NULL) -+ -+/** -+ * Power management configuration -+ * -+ * Attached value: -+ * pointer to @ref kbase_pm_callback_conf -+ * Default value: -+ * See @ref kbase_pm_callback_conf -+ */ -+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) -+extern struct kbase_pm_callback_conf pm_callbacks; -+ -+/** -+ * Platform specific configuration functions -+ * -+ * Attached value: -+ * pointer to @ref kbase_platform_funcs_conf -+ * Default value: -+ * See @ref kbase_platform_funcs_conf -+ */ -+#define PLATFORM_FUNCS (&platform_funcs) -+extern struct kbase_platform_funcs_conf platform_funcs; -+ -+/** -+ * Secure mode switch -+ * -+ * Attached value: pointer to @ref kbase_secure_ops -+ */ -+#define SECURE_CALLBACKS (NULL) -+ -diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c -new file mode 100755 -index 000000000..8ad910c12 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_config_rk.c -@@ -0,0 +1,492 @@ -+/* -+ * -+ * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ */ -+ -+/* #define ENABLE_DEBUG_LOG */ -+#include "custom_log.h" -+ -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "mali_kbase_rk.h" -+ -+/** -+ * @file mali_kbase_config_rk.c -+ * 对 platform_config_of_rk 的具体实现. -+ * -+ * mali_device_driver 包å«ä¸¤éƒ¨åˆ† : -+ * .DP : platform_dependent_part_in_mdd : -+ * ä¾èµ– platform 部分, -+ * æºç åœ¨ /platform// -+ * 在 mali_device_driver 内部, -+ * 记为 platform_dependent_part, -+ * 也被记为 platform_specific_code. -+ * .DP : common_parts_in_mdd : -+ * arm 实现的通用的部分, -+ * æºç åœ¨ / 下. -+ * 在 mali_device_driver 内部, 记为 common_parts. -+ */ -+ -+/*---------------------------------------------------------------------------*/ -+ -+#ifdef CONFIG_REGULATOR -+static int rk_pm_enable_regulator(struct kbase_device *kbdev); -+static void rk_pm_disable_regulator(struct kbase_device *kbdev); -+#else -+static inline int rk_pm_enable_regulator(struct kbase_device *kbdev) -+{ -+ return 0; -+} -+ -+static inline void rk_pm_disable_regulator(struct kbase_device *kbdev) -+{ -+} -+#endif -+ -+static int rk_pm_enable_clk(struct kbase_device *kbdev); -+ -+static void rk_pm_disable_clk(struct kbase_device *kbdev); -+ -+static int kbase_platform_rk_create_sysfs_files(struct device *dev); -+ -+static void kbase_platform_rk_remove_sysfs_files(struct device *dev); -+ -+/*---------------------------------------------------------------------------*/ -+ -+static void rk_pm_power_off_delay_work(struct work_struct *work) -+{ -+ struct rk_context *platform = -+ container_of(to_delayed_work(work), struct rk_context, work); -+ struct kbase_device *kbdev = platform->kbdev; -+ -+ if (!platform->is_powered) { -+ D("mali_dev is already powered off."); -+ return; -+ } -+ -+ if (pm_runtime_enabled(kbdev->dev)) { -+ D("to put_sync_suspend mali_dev."); -+ pm_runtime_put_sync_suspend(kbdev->dev); -+ } -+ -+ rk_pm_disable_regulator(kbdev); -+ -+ platform->is_powered = false; -+ KBASE_TIMELINE_GPU_POWER(kbdev, 0); -+ wake_unlock(&platform->wake_lock); -+} -+ -+static int kbase_platform_rk_init(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ struct rk_context *platform; -+ -+ platform = kzalloc(sizeof(*platform), GFP_KERNEL); -+ if (!platform) { -+ E("err."); -+ return -ENOMEM; -+ } -+ -+ platform->is_powered = false; -+ platform->kbdev = kbdev; -+ -+ platform->delay_ms = 200; -+ if (of_property_read_u32(kbdev->dev->of_node, "power-off-delay-ms", -+ &platform->delay_ms)) -+ W("power-off-delay-ms not available."); -+ -+ platform->power_off_wq = create_freezable_workqueue("gpu_power_off_wq"); -+ if (!platform->power_off_wq) { -+ E("couldn't create workqueue"); -+ ret = -ENOMEM; -+ goto err_wq; -+ } -+ INIT_DEFERRABLE_WORK(&platform->work, rk_pm_power_off_delay_work); -+ -+ wake_lock_init(&platform->wake_lock, WAKE_LOCK_SUSPEND, "gpu"); -+ -+ platform->utilisation_period = DEFAULT_UTILISATION_PERIOD_IN_MS; -+ -+ ret = kbase_platform_rk_create_sysfs_files(kbdev->dev); -+ if (ret) { -+ E("fail to create sysfs_files. ret = %d.", ret); -+ goto err_sysfs_files; -+ } -+ -+ kbdev->platform_context = (void *)platform; -+ pm_runtime_enable(kbdev->dev); -+ -+ return 0; -+ -+err_sysfs_files: -+ wake_lock_destroy(&platform->wake_lock); -+ destroy_workqueue(platform->power_off_wq); -+err_wq: -+ return ret; -+} -+ -+static void kbase_platform_rk_term(struct kbase_device *kbdev) -+{ -+ struct rk_context *platform = -+ (struct rk_context *)kbdev->platform_context; -+ -+ pm_runtime_disable(kbdev->dev); -+ kbdev->platform_context = NULL; -+ -+ if (platform) { -+ cancel_delayed_work_sync(&platform->work); -+ wake_lock_destroy(&platform->wake_lock); -+ destroy_workqueue(platform->power_off_wq); -+ platform->is_powered = false; -+ platform->kbdev = NULL; -+ kfree(platform); -+ } -+ kbase_platform_rk_remove_sysfs_files(kbdev->dev); -+} -+ -+struct kbase_platform_funcs_conf platform_funcs = { -+ .platform_init_func = &kbase_platform_rk_init, -+ .platform_term_func = &kbase_platform_rk_term, -+}; -+ -+/*---------------------------------------------------------------------------*/ -+ -+static int rk_pm_callback_runtime_on(struct kbase_device *kbdev) -+{ -+ return 0; -+} -+ -+static void rk_pm_callback_runtime_off(struct kbase_device *kbdev) -+{ -+} -+ -+static int rk_pm_callback_power_on(struct kbase_device *kbdev) -+{ -+ int ret = 1; /* Assume GPU has been powered off */ -+ int err = 0; -+ struct rk_context *platform = get_rk_context(kbdev); -+ -+ cancel_delayed_work_sync(&platform->work); -+ -+ err = rk_pm_enable_clk(kbdev); -+ if (err) { -+ E("failed to enable clk: %d", err); -+ return err; -+ } -+ -+ if (platform->is_powered) { -+ D("mali_device is already powered."); -+ return 0; -+ } -+ -+ /* we must enable vdd_gpu before pd_gpu_in_chip. */ -+ err = rk_pm_enable_regulator(kbdev); -+ if (err) { -+ E("fail to enable regulator, err : %d.", err); -+ return err; -+ } -+ -+ /* è‹¥ mali_dev çš„ runtime_pm 是 enabled çš„, 则... */ -+ if (pm_runtime_enabled(kbdev->dev)) { -+ D("to resume mali_dev syncly."); -+ /* 对 pd_in_chip çš„ on æ“作, -+ * 将在 pm_domain çš„ runtime_pm_callbacks 中完æˆ. -+ */ -+ err = pm_runtime_get_sync(kbdev->dev); -+ if (err < 0) { -+ E("failed to runtime resume device: %d.", err); -+ return err; -+ } else if (err == 1) { /* runtime_pm_status is still active */ -+ D("chip has NOT been powered off, no need to re-init."); -+ ret = 0; -+ } -+ } -+ -+ platform->is_powered = true; -+ KBASE_TIMELINE_GPU_POWER(kbdev, 1); -+ wake_lock(&platform->wake_lock); -+ -+ return ret; -+} -+ -+static void rk_pm_callback_power_off(struct kbase_device *kbdev) -+{ -+ struct rk_context *platform = get_rk_context(kbdev); -+ -+ rk_pm_disable_clk(kbdev); -+ queue_delayed_work(platform->power_off_wq, &platform->work, -+ msecs_to_jiffies(platform->delay_ms)); -+} -+ -+int rk_kbase_device_runtime_init(struct kbase_device *kbdev) -+{ -+ return 0; -+} -+ -+void rk_kbase_device_runtime_disable(struct kbase_device *kbdev) -+{ -+} -+ -+struct kbase_pm_callback_conf pm_callbacks = { -+ .power_on_callback = rk_pm_callback_power_on, -+ .power_off_callback = rk_pm_callback_power_off, -+#ifdef CONFIG_PM -+ .power_runtime_init_callback = rk_kbase_device_runtime_init, -+ .power_runtime_term_callback = rk_kbase_device_runtime_disable, -+ .power_runtime_on_callback = rk_pm_callback_runtime_on, -+ .power_runtime_off_callback = rk_pm_callback_runtime_off, -+#else /* CONFIG_PM */ -+ .power_runtime_init_callback = NULL, -+ .power_runtime_term_callback = NULL, -+ .power_runtime_on_callback = NULL, -+ .power_runtime_off_callback = NULL, -+#endif /* CONFIG_PM */ -+}; -+ -+int kbase_platform_early_init(void) -+{ -+ /* Nothing needed at this stage */ -+ return 0; -+} -+ -+/*---------------------------------------------------------------------------*/ -+ -+void kbase_platform_rk_shutdown(struct kbase_device *kbdev) -+{ -+ I("to make vdd_gpu enabled for turning off pd_gpu in pm_framework."); -+ rk_pm_enable_regulator(kbdev); -+} -+ -+/*---------------------------------------------------------------------------*/ -+ -+#ifdef CONFIG_REGULATOR -+static int rk_pm_enable_regulator(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ -+ if (!kbdev->regulator) { -+ W("no mali regulator control, no need to enable."); -+ goto EXIT; -+ } -+ -+ D("to enable regulator."); -+ ret = regulator_enable(kbdev->regulator); -+ if (ret) { -+ E("fail to enable regulator, ret : %d.", ret); -+ goto EXIT; -+ } -+ -+EXIT: -+ return ret; -+} -+ -+static void rk_pm_disable_regulator(struct kbase_device *kbdev) -+{ -+ if (!(kbdev->regulator)) { -+ W("no mali regulator control, no need to disable."); -+ return; -+ } -+ -+ D("to disable regulator."); -+ regulator_disable(kbdev->regulator); -+} -+#endif -+ -+static int rk_pm_enable_clk(struct kbase_device *kbdev) -+{ -+ int err = 0; -+ -+ if (!(kbdev->clock)) { -+ W("no mali clock control, no need to enable."); -+ } else { -+ D("to enable clk."); -+ err = clk_enable(kbdev->clock); -+ if (err) -+ E("failed to enable clk: %d.", err); -+ } -+ -+ return err; -+} -+ -+static void rk_pm_disable_clk(struct kbase_device *kbdev) -+{ -+ if (!(kbdev->clock)) { -+ W("no mali clock control, no need to disable."); -+ } else { -+ D("to disable clk."); -+ clk_disable(kbdev->clock); -+ } -+} -+ -+/*---------------------------------------------------------------------------*/ -+ -+static ssize_t utilisation_period_show(struct device *dev, -+ struct device_attribute *attr, -+ char *buf) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct rk_context *platform = get_rk_context(kbdev); -+ ssize_t ret = 0; -+ -+ ret += snprintf(buf, PAGE_SIZE, "%u\n", platform->utilisation_period); -+ -+ return ret; -+} -+ -+static ssize_t utilisation_period_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, -+ size_t count) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct rk_context *platform = get_rk_context(kbdev); -+ int ret = 0; -+ -+ ret = kstrtouint(buf, 0, &platform->utilisation_period); -+ if (ret) { -+ E("invalid input period : %s.", buf); -+ return ret; -+ } -+ D("set utilisation_period to '%d'.", platform->utilisation_period); -+ -+ return count; -+} -+ -+static ssize_t utilisation_show(struct device *dev, -+ struct device_attribute *attr, -+ char *buf) -+{ -+ struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct rk_context *platform = get_rk_context(kbdev); -+ ssize_t ret = 0; -+ unsigned long period_in_us = platform->utilisation_period * 1000; -+ unsigned long total_time; -+ unsigned long busy_time; -+ unsigned long utilisation; -+ -+ kbase_pm_reset_dvfs_utilisation(kbdev); -+ usleep_range(period_in_us, period_in_us + 100); -+ kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time); -+ /* 'devfreq_dev_profile' instance registered to devfreq -+ * also uses kbase_pm_reset_dvfs_utilisation -+ * and kbase_pm_get_dvfs_utilisation. -+ * it's better to cat this file when DVFS is disabled. -+ */ -+ D("total_time : %lu, busy_time : %lu.", total_time, busy_time); -+ -+ utilisation = busy_time * 100 / total_time; -+ ret += snprintf(buf, PAGE_SIZE, "%ld\n", utilisation); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR_RW(utilisation_period); -+static DEVICE_ATTR_RO(utilisation); -+ -+static int kbase_platform_rk_create_sysfs_files(struct device *dev) -+{ -+ int ret = 0; -+ -+ ret = device_create_file(dev, &dev_attr_utilisation_period); -+ if (ret) { -+ E("fail to create sysfs file 'utilisation_period'."); -+ goto out; -+ } -+ -+ ret = device_create_file(dev, &dev_attr_utilisation); -+ if (ret) { -+ E("fail to create sysfs file 'utilisation'."); -+ goto remove_utilisation_period; -+ } -+ -+ return 0; -+ -+remove_utilisation_period: -+ device_remove_file(dev, &dev_attr_utilisation_period); -+out: -+ return ret; -+} -+ -+static void kbase_platform_rk_remove_sysfs_files(struct device *dev) -+{ -+ device_remove_file(dev, &dev_attr_utilisation_period); -+ device_remove_file(dev, &dev_attr_utilisation); -+} -+ -+static int rk3288_get_soc_info(struct device *dev, struct device_node *np, -+ int *bin, int *process) -+{ -+ int ret = -EINVAL; -+ u8 value = 0; -+ char *name; -+ -+ if (!bin) -+ goto out; -+ -+ if (soc_is_rk3288w()) -+ name = "performance-w"; -+ else -+ name = "performance"; -+ if (of_property_match_string(np, "nvmem-cell-names", name) >= 0) { -+ ret = rockchip_nvmem_cell_read_u8(np, name, &value); -+ if (ret) { -+ dev_err(dev, "Failed to get soc performance value\n"); -+ goto out; -+ } -+ if (value & 0x2) -+ *bin = 3; -+ else if (value & 0x01) -+ *bin = 2; -+ else -+ *bin = 0; -+ } else { -+ dev_err(dev, "Failed to get bin config\n"); -+ } -+ if (*bin >= 0) -+ dev_info(dev, "bin=%d\n", *bin); -+ -+out: -+ return ret; -+} -+ -+static const struct rockchip_opp_data rk3288_gpu_opp_data = { -+ .get_soc_info = rk3288_get_soc_info, -+}; -+ -+static const struct of_device_id rockchip_mali_of_match[] = { -+ { -+ .compatible = "rockchip,rk3288", -+ .data = (void *)&rk3288_gpu_opp_data, -+ }, -+ { -+ .compatible = "rockchip,rk3288w", -+ .data = (void *)&rk3288_gpu_opp_data, -+ }, -+ {}, -+}; -+ -+int kbase_platform_rk_init_opp_table(struct kbase_device *kbdev) -+{ -+ rockchip_get_opp_data(rockchip_mali_of_match, &kbdev->opp_info); -+ -+ return rockchip_init_opp_table(kbdev->dev, &kbdev->opp_info, -+ "gpu_leakage", "mali"); -+} -diff --git a/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h -new file mode 100755 -index 000000000..6eab25014 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/rk/mali_kbase_rk.h -@@ -0,0 +1,62 @@ -+/* drivers/gpu/t6xx/kbase/src/platform/rk/mali_kbase_platform.h -+ * Rockchip SoC Mali-Midgard platform-dependent codes -+ * -+ * This program is free software; you can redistribute it and/or modify -+ * it under the terms of the GNU General Public License version 2 as -+ * published by the Free Software FoundatIon. -+ */ -+ -+/** -+ * @file mali_kbase_rk.h -+ * -+ * defines work_context type of platform_dependent_part. -+ */ -+ -+#ifndef _MALI_KBASE_RK_H_ -+#define _MALI_KBASE_RK_H_ -+ -+#include -+ -+/*---------------------------------------------------------------------------*/ -+ -+#define DEFAULT_UTILISATION_PERIOD_IN_MS (100) -+ -+/*---------------------------------------------------------------------------*/ -+ -+/* -+ * struct rk_context - work_context of platform_dependent_part_of_rk. -+ */ -+struct rk_context { -+ /* -+ * record the status of common_parts calling 'power_on_callback' -+ * and 'power_off_callback'. -+ */ -+ bool is_powered; -+ -+ struct kbase_device *kbdev; -+ -+ struct workqueue_struct *power_off_wq; -+ /* delayed_work_to_power_off_gpu. */ -+ struct delayed_work work; -+ unsigned int delay_ms; -+ -+ /* -+ * WAKE_LOCK_SUSPEND for ensuring to run -+ * delayed_work_to_power_off_gpu before suspend. -+ */ -+ struct wake_lock wake_lock; -+ -+ /* debug only, the period in ms to count gpu_utilisation. */ -+ unsigned int utilisation_period; -+}; -+ -+/*---------------------------------------------------------------------------*/ -+ -+static inline struct rk_context *get_rk_context( -+ const struct kbase_device *kbdev) -+{ -+ return (struct rk_context *)(kbdev->platform_context); -+} -+ -+#endif /* _MALI_KBASE_RK_H_ */ -+ -diff --git a/drivers/gpu/arm/midgard/platform/vexpress/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress/Kbuild -new file mode 100755 -index 000000000..1caa29366 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress/Kbuild -@@ -0,0 +1,18 @@ -+# -+# (C) COPYRIGHT 2012-2013, 2016 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+mali_kbase-y += \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_cpu_vexpress.o -diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h -new file mode 100755 -index 000000000..02835f129 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h -@@ -0,0 +1,75 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include "mali_kbase_cpu_vexpress.h" -+ -+/** -+ * Maximum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MAX kbase_get_platform_max_freq() -+/** -+ * Minimum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MIN kbase_get_platform_min_freq() -+ -+/** -+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock -+ * -+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_cpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) -+ -+/** -+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock -+ * -+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_gpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define GPU_SPEED_FUNC (NULL) -+ -+/** -+ * Power management configuration -+ * -+ * Attached value: pointer to @ref kbase_pm_callback_conf -+ * Default value: See @ref kbase_pm_callback_conf -+ */ -+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) -+ -+/** -+ * Platform specific configuration functions -+ * -+ * Attached value: pointer to @ref kbase_platform_funcs_conf -+ * Default value: See @ref kbase_platform_funcs_conf -+ */ -+#define PLATFORM_FUNCS (NULL) -+ -+extern struct kbase_pm_callback_conf pm_callbacks; -diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c -new file mode 100755 -index 000000000..15ce2bc5e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c -@@ -0,0 +1,85 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+#include -+#include -+#include -+#include "mali_kbase_cpu_vexpress.h" -+#include "mali_kbase_config_platform.h" -+ -+#define HARD_RESET_AT_POWER_OFF 0 -+ -+#ifndef CONFIG_OF -+static struct kbase_io_resources io_resources = { -+ .job_irq_number = 68, -+ .mmu_irq_number = 69, -+ .gpu_irq_number = 70, -+ .io_memory_region = { -+ .start = 0xFC010000, -+ .end = 0xFC010000 + (4096 * 4) - 1 -+ } -+}; -+#endif /* CONFIG_OF */ -+ -+static int pm_callback_power_on(struct kbase_device *kbdev) -+{ -+ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ -+ return 1; -+} -+ -+static void pm_callback_power_off(struct kbase_device *kbdev) -+{ -+#if HARD_RESET_AT_POWER_OFF -+ /* Cause a GPU hard reset to test whether we have actually idled the GPU -+ * and that we properly reconfigure the GPU on power up. -+ * Usually this would be dangerous, but if the GPU is working correctly it should -+ * be completely safe as the GPU should not be active at this point. -+ * However this is disabled normally because it will most likely interfere with -+ * bus logging etc. -+ */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); -+ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); -+#endif -+} -+ -+struct kbase_pm_callback_conf pm_callbacks = { -+ .power_on_callback = pm_callback_power_on, -+ .power_off_callback = pm_callback_power_off, -+ .power_suspend_callback = NULL, -+ .power_resume_callback = NULL -+}; -+ -+static struct kbase_platform_config versatile_platform_config = { -+#ifndef CONFIG_OF -+ .io_resources = &io_resources -+#endif -+}; -+ -+struct kbase_platform_config *kbase_get_platform_config(void) -+{ -+ return &versatile_platform_config; -+} -+ -+ -+int kbase_platform_early_init(void) -+{ -+ /* Nothing needed at this stage */ -+ return 0; -+} -diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c -new file mode 100755 -index 000000000..4665f98cb ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.c -@@ -0,0 +1,279 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include "mali_kbase_cpu_vexpress.h" -+ -+#define HZ_IN_MHZ (1000000) -+ -+#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) -+#define MOTHERBOARD_SYS_CFG_START (0x10000000) -+#define SYS_CFGDATA_OFFSET (0x000000A0) -+#define SYS_CFGCTRL_OFFSET (0x000000A4) -+#define SYS_CFGSTAT_OFFSET (0x000000A8) -+ -+#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) -+#define READ_REG_BIT_VALUE (0 << 30) -+#define DCC_DEFAULT_BIT_VALUE (0 << 26) -+#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) -+#define SITE_DEFAULT_BIT_VALUE (1 << 16) -+#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) -+#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) -+#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) -+#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) -+ -+#define FEED_REG_BIT_MASK (0x0F) -+#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) -+#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) -+#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) -+#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) -+#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) -+ -+/* the following three values used for reading -+ * HBI value of the LogicTile daughterboard */ -+#define VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 (0x10000000) -+#define VE_SYS_PROC_ID1_OFFSET (0x00000088) -+#define VE_LOGIC_TILE_HBI_MASK (0x00000FFF) -+ -+#define IS_SINGLE_BIT_SET(val, pos) (val&(1<> -+ FCLK_PA_DIVIDE_BIT_SHIFT); -+ /* CFGRW0[10:7] */ -+ pb_divide = ((reg_val & (FEED_REG_BIT_MASK << -+ FCLK_PB_DIVIDE_BIT_SHIFT)) >> -+ FCLK_PB_DIVIDE_BIT_SHIFT); -+ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); -+ } else if (IS_SINGLE_BIT_SET(reg_val, 1)) { -+ /* CFGRW0[1] - CLKOC */ -+ /* CFGRW0[6:3] */ -+ pa_divide = ((reg_val & (FEED_REG_BIT_MASK << -+ FCLK_PA_DIVIDE_BIT_SHIFT)) >> -+ FCLK_PA_DIVIDE_BIT_SHIFT); -+ /* CFGRW0[14:11] */ -+ pc_divide = ((reg_val & (FEED_REG_BIT_MASK << -+ FCLK_PC_DIVIDE_BIT_SHIFT)) >> -+ FCLK_PC_DIVIDE_BIT_SHIFT); -+ *cpu_clock = osc2_value * (pa_divide + 1) / (pc_divide + 1); -+ } else if (IS_SINGLE_BIT_SET(reg_val, 2)) { -+ /* CFGRW0[2] - FACLK */ -+ /* CFGRW0[18:15] */ -+ pa_divide = ((reg_val & (FEED_REG_BIT_MASK << -+ AXICLK_PA_DIVIDE_BIT_SHIFT)) >> -+ AXICLK_PA_DIVIDE_BIT_SHIFT); -+ /* CFGRW0[22:19] */ -+ pb_divide = ((reg_val & (FEED_REG_BIT_MASK << -+ AXICLK_PB_DIVIDE_BIT_SHIFT)) >> -+ AXICLK_PB_DIVIDE_BIT_SHIFT); -+ *cpu_clock = osc2_value * (pa_divide + 1) / (pb_divide + 1); -+ } else { -+ err = -EIO; -+ } -+ -+set_reg_error: -+ongoing_request: -+ raw_spin_unlock(&syscfg_lock); -+ *cpu_clock /= HZ_IN_MHZ; -+ -+ if (!err) -+ cpu_clock_speed = *cpu_clock; -+ -+ iounmap(scc_reg); -+ -+scc_reg_map_failed: -+ iounmap(syscfg_reg); -+ -+syscfg_reg_map_failed: -+ -+ return err; -+} -+ -+/** -+ * kbase_get_platform_logic_tile_type - determines which LogicTile type -+ * is used by Versatile Express -+ * -+ * When platform_config build parameter is specified as vexpress, i.e., -+ * platform_config=vexpress, GPU frequency may vary dependent on the -+ * particular platform. The GPU frequency depends on the LogicTile type. -+ * -+ * This function determines which LogicTile type is used by the platform by -+ * reading the HBI value of the daughterboard which holds the LogicTile: -+ * -+ * 0x217 HBI0217 Virtex-6 -+ * 0x192 HBI0192 Virtex-5 -+ * 0x247 HBI0247 Virtex-7 -+ * -+ * Return: HBI value of the logic tile daughterboard, zero if not accessible -+ */ -+static u32 kbase_get_platform_logic_tile_type(void) -+{ -+ void __iomem *syscfg_reg = NULL; -+ u32 sys_procid1 = 0; -+ -+ syscfg_reg = ioremap(VE_MOTHERBOARD_PERIPHERALS_SMB_CS7 + VE_SYS_PROC_ID1_OFFSET, 4); -+ if (NULL != syscfg_reg) { -+ sys_procid1 = readl(syscfg_reg); -+ iounmap(syscfg_reg); -+ } -+ -+ return sys_procid1 & VE_LOGIC_TILE_HBI_MASK; -+} -+ -+u32 kbase_get_platform_min_freq(void) -+{ -+ u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); -+ -+ switch (ve_logic_tile) { -+ case 0x217: -+ /* Virtex 6, HBI0217 */ -+ return VE_VIRTEX6_GPU_FREQ_MIN; -+ case 0x247: -+ /* Virtex 7, HBI0247 */ -+ return VE_VIRTEX7_GPU_FREQ_MIN; -+ default: -+ /* all other logic tiles, i.e., Virtex 5 HBI0192 -+ * or unsuccessful reading from the platform - -+ * fall back to some default value */ -+ return VE_DEFAULT_GPU_FREQ_MIN; -+ } -+} -+ -+u32 kbase_get_platform_max_freq(void) -+{ -+ u32 ve_logic_tile = kbase_get_platform_logic_tile_type(); -+ -+ switch (ve_logic_tile) { -+ case 0x217: -+ /* Virtex 6, HBI0217 */ -+ return VE_VIRTEX6_GPU_FREQ_MAX; -+ case 0x247: -+ /* Virtex 7, HBI0247 */ -+ return VE_VIRTEX7_GPU_FREQ_MAX; -+ default: -+ /* all other logic tiles, i.e., Virtex 5 HBI0192 -+ * or unsuccessful reading from the platform - -+ * fall back to some default value */ -+ return VE_DEFAULT_GPU_FREQ_MAX; -+ } -+} -diff --git a/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h -new file mode 100755 -index 000000000..da8656981 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_cpu_vexpress.h -@@ -0,0 +1,38 @@ -+/* -+ * -+ * (C) COPYRIGHT 2012-2013, 2015-2016 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#ifndef _KBASE_CPU_VEXPRESS_H_ -+#define _KBASE_CPU_VEXPRESS_H_ -+ -+/** -+ * Versatile Express implementation of @ref kbase_cpu_clk_speed_func. -+ */ -+int kbase_get_vexpress_cpu_clock_speed(u32 *cpu_clock); -+ -+/** -+ * Get the minimum GPU frequency for the attached logic tile -+ */ -+u32 kbase_get_platform_min_freq(void); -+ -+/** -+ * Get the maximum GPU frequency for the attached logic tile -+ */ -+u32 kbase_get_platform_max_freq(void); -+ -+#endif /* _KBASE_CPU_VEXPRESS_H_ */ -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild -new file mode 100755 -index 000000000..7efe8fa42 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild -@@ -0,0 +1,16 @@ -+# -+# (C) COPYRIGHT 2013-2014, 2016 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+mali_kbase-y += $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h -new file mode 100755 -index 000000000..0efbf3962 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h -@@ -0,0 +1,73 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/** -+ * Maximum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MAX 5000 -+/** -+ * Minimum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MIN 5000 -+ -+/** -+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock -+ * -+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_cpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define CPU_SPEED_FUNC (&kbase_cpuprops_get_default_clock_speed) -+ -+/** -+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock -+ * -+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_gpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define GPU_SPEED_FUNC (NULL) -+ -+/** -+ * Power management configuration -+ * -+ * Attached value: pointer to @ref kbase_pm_callback_conf -+ * Default value: See @ref kbase_pm_callback_conf -+ */ -+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) -+ -+/** -+ * Platform specific configuration functions -+ * -+ * Attached value: pointer to @ref kbase_platform_funcs_conf -+ * Default value: See @ref kbase_platform_funcs_conf -+ */ -+#define PLATFORM_FUNCS (NULL) -+ -+extern struct kbase_pm_callback_conf pm_callbacks; -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c -new file mode 100755 -index 000000000..3ff0930fb ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c -@@ -0,0 +1,79 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+#include -+ -+#define HARD_RESET_AT_POWER_OFF 0 -+ -+#ifndef CONFIG_OF -+static struct kbase_io_resources io_resources = { -+ .job_irq_number = 68, -+ .mmu_irq_number = 69, -+ .gpu_irq_number = 70, -+ .io_memory_region = { -+ .start = 0x2f010000, -+ .end = 0x2f010000 + (4096 * 4) - 1} -+}; -+#endif -+ -+static int pm_callback_power_on(struct kbase_device *kbdev) -+{ -+ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ -+ return 1; -+} -+ -+static void pm_callback_power_off(struct kbase_device *kbdev) -+{ -+#if HARD_RESET_AT_POWER_OFF -+ /* Cause a GPU hard reset to test whether we have actually idled the GPU -+ * and that we properly reconfigure the GPU on power up. -+ * Usually this would be dangerous, but if the GPU is working correctly it should -+ * be completely safe as the GPU should not be active at this point. -+ * However this is disabled normally because it will most likely interfere with -+ * bus logging etc. -+ */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); -+ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); -+#endif -+} -+ -+struct kbase_pm_callback_conf pm_callbacks = { -+ .power_on_callback = pm_callback_power_on, -+ .power_off_callback = pm_callback_power_off, -+ .power_suspend_callback = NULL, -+ .power_resume_callback = NULL -+}; -+ -+static struct kbase_platform_config versatile_platform_config = { -+#ifndef CONFIG_OF -+ .io_resources = &io_resources -+#endif -+}; -+ -+struct kbase_platform_config *kbase_get_platform_config(void) -+{ -+ return &versatile_platform_config; -+} -+ -+int kbase_platform_early_init(void) -+{ -+ /* Nothing needed at this stage */ -+ return 0; -+} -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild -new file mode 100755 -index 000000000..1caa29366 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild -@@ -0,0 +1,18 @@ -+# -+# (C) COPYRIGHT 2012-2013, 2016 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+mali_kbase-y += \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_config_vexpress.o \ -+ $(MALI_PLATFORM_THIRDPARTY_DIR)/mali_kbase_cpu_vexpress.o -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h -new file mode 100755 -index 000000000..dbdf21e00 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h -@@ -0,0 +1,75 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include "mali_kbase_cpu_vexpress.h" -+ -+/** -+ * Maximum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MAX 10000 -+/** -+ * Minimum frequency GPU will be clocked at. Given in kHz. -+ * This must be specified as there is no default value. -+ * -+ * Attached value: number in kHz -+ * Default value: NA -+ */ -+#define GPU_FREQ_KHZ_MIN 10000 -+ -+/** -+ * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock -+ * -+ * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_cpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define CPU_SPEED_FUNC (&kbase_get_vexpress_cpu_clock_speed) -+ -+/** -+ * GPU_SPEED_FUNC - A pointer to a function that calculates the GPU clock -+ * -+ * GPU clock speed of the platform in MHz - see kbase_gpu_clk_speed_func -+ * for the function prototype. -+ * -+ * Attached value: A kbase_gpu_clk_speed_func. -+ * Default Value: NA -+ */ -+#define GPU_SPEED_FUNC (NULL) -+ -+/** -+ * Power management configuration -+ * -+ * Attached value: pointer to @ref kbase_pm_callback_conf -+ * Default value: See @ref kbase_pm_callback_conf -+ */ -+#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) -+ -+/** -+ * Platform specific configuration functions -+ * -+ * Attached value: pointer to @ref kbase_platform_funcs_conf -+ * Default value: See @ref kbase_platform_funcs_conf -+ */ -+#define PLATFORM_FUNCS (NULL) -+ -+extern struct kbase_pm_callback_conf pm_callbacks; -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c -new file mode 100755 -index 000000000..76ffe4a1e ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c -@@ -0,0 +1,83 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2014 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+#include -+#include -+#include -+#include "mali_kbase_cpu_vexpress.h" -+ -+#define HARD_RESET_AT_POWER_OFF 0 -+ -+#ifndef CONFIG_OF -+static struct kbase_io_resources io_resources = { -+ .job_irq_number = 75, -+ .mmu_irq_number = 76, -+ .gpu_irq_number = 77, -+ .io_memory_region = { -+ .start = 0x2F000000, -+ .end = 0x2F000000 + (4096 * 4) - 1} -+}; -+#endif -+ -+static int pm_callback_power_on(struct kbase_device *kbdev) -+{ -+ /* Nothing is needed on VExpress, but we may have destroyed GPU state (if the below HARD_RESET code is active) */ -+ return 1; -+} -+ -+static void pm_callback_power_off(struct kbase_device *kbdev) -+{ -+#if HARD_RESET_AT_POWER_OFF -+ /* Cause a GPU hard reset to test whether we have actually idled the GPU -+ * and that we properly reconfigure the GPU on power up. -+ * Usually this would be dangerous, but if the GPU is working correctly it should -+ * be completely safe as the GPU should not be active at this point. -+ * However this is disabled normally because it will most likely interfere with -+ * bus logging etc. -+ */ -+ KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0); -+ kbase_os_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET); -+#endif -+} -+ -+struct kbase_pm_callback_conf pm_callbacks = { -+ .power_on_callback = pm_callback_power_on, -+ .power_off_callback = pm_callback_power_off, -+ .power_suspend_callback = NULL, -+ .power_resume_callback = NULL -+}; -+ -+static struct kbase_platform_config versatile_platform_config = { -+#ifndef CONFIG_OF -+ .io_resources = &io_resources -+#endif -+}; -+ -+struct kbase_platform_config *kbase_get_platform_config(void) -+{ -+ return &versatile_platform_config; -+} -+ -+int kbase_platform_early_init(void) -+{ -+ /* Nothing needed at this stage */ -+ return 0; -+} -+ -diff --git a/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c -new file mode 100755 -index 000000000..816dff498 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_cpu_vexpress.c -@@ -0,0 +1,71 @@ -+/* -+ * -+ * (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+ -+ -+#include -+#include -+#include "mali_kbase_cpu_vexpress.h" -+ -+#define HZ_IN_MHZ (1000000) -+ -+#define CORETILE_EXPRESS_A9X4_SCC_START (0x100E2000) -+#define MOTHERBOARD_SYS_CFG_START (0x10000000) -+#define SYS_CFGDATA_OFFSET (0x000000A0) -+#define SYS_CFGCTRL_OFFSET (0x000000A4) -+#define SYS_CFGSTAT_OFFSET (0x000000A8) -+ -+#define SYS_CFGCTRL_START_BIT_VALUE (1 << 31) -+#define READ_REG_BIT_VALUE (0 << 30) -+#define DCC_DEFAULT_BIT_VALUE (0 << 26) -+#define SYS_CFG_OSC_FUNC_BIT_VALUE (1 << 20) -+#define SITE_DEFAULT_BIT_VALUE (1 << 16) -+#define BOARD_STACK_POS_DEFAULT_BIT_VALUE (0 << 12) -+#define DEVICE_DEFAULT_BIT_VALUE (2 << 0) -+#define SYS_CFG_COMPLETE_BIT_VALUE (1 << 0) -+#define SYS_CFG_ERROR_BIT_VALUE (1 << 1) -+ -+#define FEED_REG_BIT_MASK (0x0F) -+#define FCLK_PA_DIVIDE_BIT_SHIFT (0x03) -+#define FCLK_PB_DIVIDE_BIT_SHIFT (0x07) -+#define FCLK_PC_DIVIDE_BIT_SHIFT (0x0B) -+#define AXICLK_PA_DIVIDE_BIT_SHIFT (0x0F) -+#define AXICLK_PB_DIVIDE_BIT_SHIFT (0x13) -+ -+#define IS_SINGLE_BIT_SET(val, pos) (val&(1< -+ -+/** -+ * @addtogroup uk_api User-Kernel Interface API -+ * @{ -+ */ -+ -+/** -+ * @addtogroup uk_api_kernel UKK (Kernel side) -+ * @{ -+ */ -+ -+/** -+ * Internal OS specific data structure associated with each UKK session. Part -+ * of a ukk_session object. -+ */ -+typedef struct ukkp_session { -+ int dummy; /**< No internal OS specific data at this time */ -+} ukkp_session; -+ -+/** @} end group uk_api_kernel */ -+ -+/** @} end group uk_api */ -+ -+#endif /* _UKK_OS_H__ */ -diff --git a/drivers/gpu/arm/midgard/protected_mode_switcher.h b/drivers/gpu/arm/midgard/protected_mode_switcher.h -new file mode 100755 -index 000000000..5dc2f3ba8 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/protected_mode_switcher.h -@@ -0,0 +1,64 @@ -+/* -+ * -+ * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _PROTECTED_MODE_SWITCH_H_ -+#define _PROTECTED_MODE_SWITCH_H_ -+ -+struct protected_mode_device; -+ -+/** -+ * struct protected_mode_ops - Callbacks for protected mode switch operations -+ * -+ * @protected_mode_enable: Callback to enable protected mode for device -+ * @protected_mode_disable: Callback to disable protected mode for device -+ */ -+struct protected_mode_ops { -+ /** -+ * protected_mode_enable() - Enable protected mode on device -+ * @dev: The struct device -+ * -+ * Return: 0 on success, non-zero on error -+ */ -+ int (*protected_mode_enable)( -+ struct protected_mode_device *protected_dev); -+ -+ /** -+ * protected_mode_disable() - Disable protected mode on device, and -+ * reset device -+ * @dev: The struct device -+ * -+ * Return: 0 on success, non-zero on error -+ */ -+ int (*protected_mode_disable)( -+ struct protected_mode_device *protected_dev); -+}; -+ -+/** -+ * struct protected_mode_device - Device structure for protected mode devices -+ * -+ * @ops - Callbacks associated with this device -+ * @data - Pointer to device private data -+ * -+ * This structure should be registered with the platform device using -+ * platform_set_drvdata(). -+ */ -+struct protected_mode_device { -+ struct protected_mode_ops ops; -+ void *data; -+}; -+ -+#endif /* _PROTECTED_MODE_SWITCH_H_ */ -diff --git a/drivers/gpu/arm/midgard/rename.h b/drivers/gpu/arm/midgard/rename.h -new file mode 100755 -index 000000000..c94b67ff2 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/rename.h -@@ -0,0 +1,422 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+#ifndef _RENAME_H -+#define _RENAME_H -+#define __crc_kbase_create_context midgard___crc_kbase_create_context -+#define __crc_kbase_destroy_context midgard___crc_kbase_destroy_context -+#define __crc_kbase_find_device midgard___crc_kbase_find_device -+#define __crc_kbase_instr_hwcnt_clear midgard___crc_kbase_instr_hwcnt_clear -+#define __crc_kbase_instr_hwcnt_dump_complete midgard___crc_kbase_instr_hwcnt_dump_complete -+#define __crc_kbase_instr_hwcnt_request_dump midgard___crc_kbase_instr_hwcnt_request_dump -+#define __crc_kbase_release_device midgard___crc_kbase_release_device -+#define jd_done_nolock midgard_jd_done_nolock -+#define kbase_add_va_region midgard_kbase_add_va_region -+#define kbase_alloc_free_region midgard_kbase_alloc_free_region -+#define kbase_alloc_phy_pages_helper midgard_kbase_alloc_phy_pages_helper -+#define kbase_alloc_phy_pages midgard_kbase_alloc_phy_pages -+#define kbase_as_fault_debugfs_init midgard_kbase_as_fault_debugfs_init -+#define kbase_backend_complete_wq midgard_kbase_backend_complete_wq -+#define kbase_backend_complete_wq_post_sched midgard_kbase_backend_complete_wq_post_sched -+#define kbase_backend_ctx_count_changed midgard_kbase_backend_ctx_count_changed -+#define kbase_backend_find_and_release_free_address_space midgard_kbase_backend_find_and_release_free_address_space -+#define kbase_backend_get_current_flush_id midgard_kbase_backend_get_current_flush_id -+#define kbase_backend_get_gpu_time midgard_kbase_backend_get_gpu_time -+#define kbase_backend_gpuprops_get_features midgard_kbase_backend_gpuprops_get_features -+#define kbase_backend_gpuprops_get midgard_kbase_backend_gpuprops_get -+#define kbase_backend_inspect_tail midgard_kbase_backend_inspect_tail -+#define kbase_backend_nr_atoms_on_slot midgard_kbase_backend_nr_atoms_on_slot -+#define kbase_backend_nr_atoms_submitted midgard_kbase_backend_nr_atoms_submitted -+#define kbase_backend_release_ctx_irq midgard_kbase_backend_release_ctx_irq -+#define kbase_backend_release_ctx_noirq midgard_kbase_backend_release_ctx_noirq -+#define kbase_backend_reset midgard_kbase_backend_reset -+#define kbase_backend_run_atom midgard_kbase_backend_run_atom -+#define kbase_backend_slot_free midgard_kbase_backend_slot_free -+#define kbase_backend_slot_update midgard_kbase_backend_slot_update -+#define kbase_backend_soft_hard_stop_slot midgard_kbase_backend_soft_hard_stop_slot -+#define kbase_backend_timeouts_changed midgard_kbase_backend_timeouts_changed -+#define kbase_backend_timer_init midgard_kbase_backend_timer_init -+#define kbase_backend_timer_resume midgard_kbase_backend_timer_resume -+#define kbase_backend_timer_suspend midgard_kbase_backend_timer_suspend -+#define kbase_backend_timer_term midgard_kbase_backend_timer_term -+#define kbase_backend_use_ctx midgard_kbase_backend_use_ctx -+#define kbase_backend_use_ctx_sched midgard_kbase_backend_use_ctx_sched -+#define kbase_cache_enabled midgard_kbase_cache_enabled -+#define kbase_cache_set_coherency_mode midgard_kbase_cache_set_coherency_mode -+#define kbase_cancel_soft_job midgard_kbase_cancel_soft_job -+#define kbase_check_alloc_flags midgard_kbase_check_alloc_flags -+#define kbase_check_import_flags midgard_kbase_check_import_flags -+#define kbase_clean_caches_done midgard_kbase_clean_caches_done -+#define kbase_create_context midgard_kbase_create_context -+#define kbase_ctx_sched_init midgard_kbase_ctx_sched_init -+#define kbase_ctx_sched_release_ctx midgard_kbase_ctx_sched_release_ctx -+#define kbase_ctx_sched_remove_ctx midgard_kbase_ctx_sched_remove_ctx -+#define kbase_ctx_sched_restore_all_as midgard_kbase_ctx_sched_restore_all_as -+#define kbase_ctx_sched_retain_ctx midgard_kbase_ctx_sched_retain_ctx -+#define kbase_ctx_sched_retain_ctx_refcount midgard_kbase_ctx_sched_retain_ctx_refcount -+#define kbase_ctx_sched_term midgard_kbase_ctx_sched_term -+#define kbase_debug_assert_register_hook midgard_kbase_debug_assert_register_hook -+#define kbase_debug_job_fault_context_init midgard_kbase_debug_job_fault_context_init -+#define kbase_debug_job_fault_context_term midgard_kbase_debug_job_fault_context_term -+#define kbase_debug_job_fault_debugfs_init midgard_kbase_debug_job_fault_debugfs_init -+#define kbase_debug_job_fault_dev_init midgard_kbase_debug_job_fault_dev_init -+#define kbase_debug_job_fault_dev_term midgard_kbase_debug_job_fault_dev_term -+#define kbase_debug_job_fault_process midgard_kbase_debug_job_fault_process -+#define kbase_debug_job_fault_reg_snapshot_init midgard_kbase_debug_job_fault_reg_snapshot_init -+#define kbase_debug_mem_view_init midgard_kbase_debug_mem_view_init -+#define kbase_destroy_context midgard_kbase_destroy_context -+#define kbase_devfreq_init midgard_kbase_devfreq_init -+#define kbase_devfreq_set_core_mask midgard_kbase_devfreq_set_core_mask -+#define kbase_devfreq_term midgard_kbase_devfreq_term -+#define kbase_device_alloc midgard_kbase_device_alloc -+#define kbase_device_free midgard_kbase_device_free -+#define kbase_device_init midgard_kbase_device_init -+#define kbase_device_term midgard_kbase_device_term -+#define kbase_disjoint_event_get midgard_kbase_disjoint_event_get -+#define kbase_disjoint_event midgard_kbase_disjoint_event -+#define kbase_disjoint_event_potential midgard_kbase_disjoint_event_potential -+#define kbase_disjoint_init midgard_kbase_disjoint_init -+#define kbase_disjoint_state_down midgard_kbase_disjoint_state_down -+#define kbase_disjoint_state_up midgard_kbase_disjoint_state_up -+#define kbase_drv_name midgard_kbase_drv_name -+#define kbase_event_cleanup midgard_kbase_event_cleanup -+#define kbase_event_close midgard_kbase_event_close -+#define kbase_event_dequeue midgard_kbase_event_dequeue -+#define kbase_event_init midgard_kbase_event_init -+#define kbase_event_pending midgard_kbase_event_pending -+#define kbase_event_post midgard_kbase_event_post -+#define kbase_event_wakeup midgard_kbase_event_wakeup -+#define kbase_fence_add_callback midgard_kbase_fence_add_callback -+#define kbase_fence_free_callbacks midgard_kbase_fence_free_callbacks -+#define kbase_fence_ops midgard_kbase_fence_ops -+#define kbase_fence_out_new midgard_kbase_fence_out_new -+#define kbase_find_device midgard_kbase_find_device -+#define kbase_finish_soft_job midgard_kbase_finish_soft_job -+#define kbase_flush_mmu_wqs midgard_kbase_flush_mmu_wqs -+#define kbase_free_alloced_region midgard_kbase_free_alloced_region -+#define kbase_free_phy_pages_helper midgard_kbase_free_phy_pages_helper -+#define kbase_get_real_power midgard_kbase_get_real_power -+#define kbase_gpu_complete_hw midgard_kbase_gpu_complete_hw -+#define kbase_gpu_dump_slots midgard_kbase_gpu_dump_slots -+#define kbase_gpu_inspect midgard_kbase_gpu_inspect -+#define kbase_gpu_interrupt midgard_kbase_gpu_interrupt -+#define kbase_gpu_irq_evict midgard_kbase_gpu_irq_evict -+#define kbase_gpu_mmap midgard_kbase_gpu_mmap -+#define kbase_gpu_munmap midgard_kbase_gpu_munmap -+#define kbase_gpuprops_populate_user_buffer midgard_kbase_gpuprops_populate_user_buffer -+#define kbase_gpuprops_set_features midgard_kbase_gpuprops_set_features -+#define kbase_gpuprops_set midgard_kbase_gpuprops_set -+#define kbase_gpuprops_update_core_props_gpu_id midgard_kbase_gpuprops_update_core_props_gpu_id -+#define kbase_gpu_vm_lock midgard_kbase_gpu_vm_lock -+#define kbase_gpu_vm_unlock midgard_kbase_gpu_vm_unlock -+#define kbase_hwaccess_pm_gpu_active midgard_kbase_hwaccess_pm_gpu_active -+#define kbase_hwaccess_pm_gpu_idle midgard_kbase_hwaccess_pm_gpu_idle -+#define kbase_hwaccess_pm_halt midgard_kbase_hwaccess_pm_halt -+#define kbase_hwaccess_pm_init midgard_kbase_hwaccess_pm_init -+#define kbase_hwaccess_pm_powerup midgard_kbase_hwaccess_pm_powerup -+#define kbase_hwaccess_pm_resume midgard_kbase_hwaccess_pm_resume -+#define kbase_hwaccess_pm_suspend midgard_kbase_hwaccess_pm_suspend -+#define kbase_hwaccess_pm_term midgard_kbase_hwaccess_pm_term -+#define kbase_hw_set_features_mask midgard_kbase_hw_set_features_mask -+#define kbase_hw_set_issues_mask midgard_kbase_hw_set_issues_mask -+#define kbase_install_interrupts midgard_kbase_install_interrupts -+#define kbase_instr_backend_init midgard_kbase_instr_backend_init -+#define kbase_instr_backend_term midgard_kbase_instr_backend_term -+#define kbase_instr_hwcnt_clear midgard_kbase_instr_hwcnt_clear -+#define kbase_instr_hwcnt_disable_internal midgard_kbase_instr_hwcnt_disable_internal -+#define kbase_instr_hwcnt_dump_complete midgard_kbase_instr_hwcnt_dump_complete -+#define kbase_instr_hwcnt_enable_internal midgard_kbase_instr_hwcnt_enable_internal -+#define kbase_instr_hwcnt_request_dump midgard_kbase_instr_hwcnt_request_dump -+#define kbase_instr_hwcnt_sample_done midgard_kbase_instr_hwcnt_sample_done -+#define kbase_instr_hwcnt_wait_for_dump midgard_kbase_instr_hwcnt_wait_for_dump -+#define kbase_invoke_smc_fid midgard_kbase_invoke_smc_fid -+#define kbase_invoke_smc midgard_kbase_invoke_smc -+#define kbase_io_history_dump midgard_kbase_io_history_dump -+#define kbase_io_history_init midgard_kbase_io_history_init -+#define kbase_io_history_term midgard_kbase_io_history_term -+#define kbase_ipa_debugfs_init midgard_kbase_ipa_debugfs_init -+#define kbase_ipa_init midgard_kbase_ipa_init -+#define kbase_ipa_init_model midgard_kbase_ipa_init_model -+#define kbase_ipa_model_add_param_s32 midgard_kbase_ipa_model_add_param_s32 -+#define kbase_ipa_model_add_param_string midgard_kbase_ipa_model_add_param_string -+#define kbase_ipa_model_name_from_id midgard_kbase_ipa_model_name_from_id -+#define kbase_ipa_model_param_add midgard_kbase_ipa_model_param_add -+#define kbase_ipa_model_param_free_all midgard_kbase_ipa_model_param_free_all -+#define kbase_ipa_model_recalculate midgard_kbase_ipa_model_recalculate -+#define kbase_ipa_power_model_ops midgard_kbase_ipa_power_model_ops -+#define kbase_ipa_term midgard_kbase_ipa_term -+#define kbase_ipa_term_model midgard_kbase_ipa_term_model -+#define kbase_jd_cancel midgard_kbase_jd_cancel -+#define kbase_jd_done midgard_kbase_jd_done -+#define kbase_jd_done_worker midgard_kbase_jd_done_worker -+#define kbase_jd_exit midgard_kbase_jd_exit -+#define kbase_jd_free_external_resources midgard_kbase_jd_free_external_resources -+#define kbase_jd_init midgard_kbase_jd_init -+#define kbase_jd_submit midgard_kbase_jd_submit -+#define kbase_jd_zap_context midgard_kbase_jd_zap_context -+#define kbase_jit_allocate midgard_kbase_jit_allocate -+#define kbase_jit_backing_lost midgard_kbase_jit_backing_lost -+#define kbase_jit_debugfs_init midgard_kbase_jit_debugfs_init -+#define kbase_jit_evict midgard_kbase_jit_evict -+#define kbase_jit_free midgard_kbase_jit_free -+#define kbase_jit_init midgard_kbase_jit_init -+#define kbase_jit_term midgard_kbase_jit_term -+#define kbase_jm_complete midgard_kbase_jm_complete -+#define kbase_jm_idle_ctx midgard_kbase_jm_idle_ctx -+#define kbase_jm_kick midgard_kbase_jm_kick -+#define kbase_jm_return_atom_to_js midgard_kbase_jm_return_atom_to_js -+#define kbase_jm_try_kick_all midgard_kbase_jm_try_kick_all -+#define kbase_jm_try_kick midgard_kbase_jm_try_kick -+#define kbase_jm_wait_for_zero_jobs midgard_kbase_jm_wait_for_zero_jobs -+#define kbase_job_check_enter_disjoint midgard_kbase_job_check_enter_disjoint -+#define kbase_job_check_leave_disjoint midgard_kbase_job_check_leave_disjoint -+#define kbase_job_done midgard_kbase_job_done -+#define kbase_job_fault_get_reg_snapshot midgard_kbase_job_fault_get_reg_snapshot -+#define kbase_job_hw_submit midgard_kbase_job_hw_submit -+#define kbase_job_slot_ctx_priority_check_locked midgard_kbase_job_slot_ctx_priority_check_locked -+#define kbase_job_slot_halt midgard_kbase_job_slot_halt -+#define kbase_job_slot_hardstop midgard_kbase_job_slot_hardstop -+#define kbase_job_slot_init midgard_kbase_job_slot_init -+#define kbase_job_slot_softstop midgard_kbase_job_slot_softstop -+#define kbase_job_slot_softstop_swflags midgard_kbase_job_slot_softstop_swflags -+#define kbase_job_slot_term midgard_kbase_job_slot_term -+#define kbase_js_complete_atom midgard_kbase_js_complete_atom -+#define kbase_js_complete_atom_wq midgard_kbase_js_complete_atom_wq -+#define kbase_js_dep_resolved_submit midgard_kbase_js_dep_resolved_submit -+#define kbase_js_is_atom_valid midgard_kbase_js_is_atom_valid -+#define kbase_js_pull midgard_kbase_js_pull -+#define kbase_js_sched midgard_kbase_js_sched -+#define kbase_js_set_timeouts midgard_kbase_js_set_timeouts -+#define kbase_js_unpull midgard_kbase_js_unpull -+#define kbase_js_zap_context midgard_kbase_js_zap_context -+#define kbase_map_external_resource midgard_kbase_map_external_resource -+#define kbase_mem_alias midgard_kbase_mem_alias -+#define kbase_mem_alloc midgard_kbase_mem_alloc -+#define kbase_mem_alloc_page midgard_kbase_mem_alloc_page -+#define kbase_mem_commit midgard_kbase_mem_commit -+#define kbase_mem_evictable_deinit midgard_kbase_mem_evictable_deinit -+#define kbase_mem_evictable_init midgard_kbase_mem_evictable_init -+#define kbase_mem_evictable_make midgard_kbase_mem_evictable_make -+#define kbase_mem_evictable_unmake midgard_kbase_mem_evictable_unmake -+#define kbase_mem_flags_change midgard_kbase_mem_flags_change -+#define kbase_mem_free midgard_kbase_mem_free -+#define kbase_mem_free_region midgard_kbase_mem_free_region -+#define kbase_mem_grow_gpu_mapping midgard_kbase_mem_grow_gpu_mapping -+#define kbase_mem_halt midgard_kbase_mem_halt -+#define kbase_mem_import midgard_kbase_mem_import -+#define kbase_mem_init midgard_kbase_mem_init -+#define kbase_mem_kref_free midgard_kbase_mem_kref_free -+#define kbase_mem_pool_alloc midgard_kbase_mem_pool_alloc -+#define kbase_mem_pool_alloc_pages midgard_kbase_mem_pool_alloc_pages -+#define kbase_mem_pool_debugfs_init midgard_kbase_mem_pool_debugfs_init -+#define kbase_mem_pool_free midgard_kbase_mem_pool_free -+#define kbase_mem_pool_free_pages midgard_kbase_mem_pool_free_pages -+#define kbase_mem_pool_grow midgard_kbase_mem_pool_grow -+#define kbase_mem_pool_init midgard_kbase_mem_pool_init -+#define kbase_mem_pool_set_max_size midgard_kbase_mem_pool_set_max_size -+#define kbase_mem_pool_term midgard_kbase_mem_pool_term -+#define kbase_mem_pool_trim midgard_kbase_mem_pool_trim -+#define kbase_mem_query midgard_kbase_mem_query -+#define kbase_mem_term midgard_kbase_mem_term -+#define kbase_mmu_disable_as midgard_kbase_mmu_disable_as -+#define kbase_mmu_disable midgard_kbase_mmu_disable -+#define kbase_mmu_dump midgard_kbase_mmu_dump -+#define kbase_mmu_hw_clear_fault midgard_kbase_mmu_hw_clear_fault -+#define kbase_mmu_hw_configure midgard_kbase_mmu_hw_configure -+#define kbase_mmu_hw_do_operation midgard_kbase_mmu_hw_do_operation -+#define kbase_mmu_hw_enable_fault midgard_kbase_mmu_hw_enable_fault -+#define kbase_mmu_init midgard_kbase_mmu_init -+#define kbase_mmu_insert_pages midgard_kbase_mmu_insert_pages -+#define kbase_mmu_insert_pages_no_flush midgard_kbase_mmu_insert_pages_no_flush -+#define kbase_mmu_insert_single_page midgard_kbase_mmu_insert_single_page -+#define kbase_mmu_interrupt midgard_kbase_mmu_interrupt -+#define kbase_mmu_mode_get_aarch64 midgard_kbase_mmu_mode_get_aarch64 -+#define kbase_mmu_mode_get_lpae midgard_kbase_mmu_mode_get_lpae -+#define kbase_mmu_teardown_pages midgard_kbase_mmu_teardown_pages -+#define kbase_mmu_term midgard_kbase_mmu_term -+#define kbase_mmu_update midgard_kbase_mmu_update -+#define kbase_mmu_update_pages midgard_kbase_mmu_update_pages -+#define kbase_os_mem_map_lock midgard_kbase_os_mem_map_lock -+#define kbase_os_mem_map_unlock midgard_kbase_os_mem_map_unlock -+#define kbasep_cache_clean_worker midgard_kbasep_cache_clean_worker -+#define kbasep_common_test_interrupt_handlers midgard_kbasep_common_test_interrupt_handlers -+#define kbasep_complete_triggered_soft_events midgard_kbasep_complete_triggered_soft_events -+#define kbasep_debug_assert_call_hook midgard_kbasep_debug_assert_call_hook -+#define kbasep_find_enclosing_cpu_mapping_offset midgard_kbasep_find_enclosing_cpu_mapping_offset -+#define kbasep_gpu_memory_debugfs_init midgard_kbasep_gpu_memory_debugfs_init -+#define kbasep_jd_debugfs_ctx_init midgard_kbasep_jd_debugfs_ctx_init -+#define kbasep_job_slot_soft_or_hard_stop_do_action midgard_kbasep_job_slot_soft_or_hard_stop_do_action -+#define kbasep_js_add_job midgard_kbasep_js_add_job -+#define kbasep_js_atom_priority_to_relative midgard_kbasep_js_atom_priority_to_relative -+#define kbasep_js_ctx_attr_ctx_release_atom midgard_kbasep_js_ctx_attr_ctx_release_atom -+#define kbasep_js_ctx_attr_ctx_retain_atom midgard_kbasep_js_ctx_attr_ctx_retain_atom -+#define kbasep_js_ctx_attr_runpool_release_ctx midgard_kbasep_js_ctx_attr_runpool_release_ctx -+#define kbasep_js_ctx_attr_runpool_retain_ctx midgard_kbasep_js_ctx_attr_runpool_retain_ctx -+#define kbasep_js_devdata_halt midgard_kbasep_js_devdata_halt -+#define kbasep_js_devdata_init midgard_kbasep_js_devdata_init -+#define kbasep_js_devdata_term midgard_kbasep_js_devdata_term -+#define kbasep_js_kctx_init midgard_kbasep_js_kctx_init -+#define kbasep_js_kctx_term midgard_kbasep_js_kctx_term -+#define kbasep_js_relative_priority_to_atom midgard_kbasep_js_relative_priority_to_atom -+#define kbasep_js_release_privileged_ctx midgard_kbasep_js_release_privileged_ctx -+#define kbasep_js_remove_cancelled_job midgard_kbasep_js_remove_cancelled_job -+#define kbasep_js_remove_job midgard_kbasep_js_remove_job -+#define kbasep_js_resume midgard_kbasep_js_resume -+#define kbasep_js_runpool_release_ctx_and_katom_retained_state midgard_kbasep_js_runpool_release_ctx_and_katom_retained_state -+#define kbasep_js_runpool_release_ctx midgard_kbasep_js_runpool_release_ctx -+#define kbasep_js_runpool_release_ctx_nolock midgard_kbasep_js_runpool_release_ctx_nolock -+#define kbasep_js_runpool_requeue_or_kill_ctx midgard_kbasep_js_runpool_requeue_or_kill_ctx -+#define kbasep_js_schedule_privileged_ctx midgard_kbasep_js_schedule_privileged_ctx -+#define kbasep_js_suspend midgard_kbasep_js_suspend -+#define kbase_platform_early_init midgard_kbase_platform_early_init -+#define kbase_platform_rk_init_opp_table midgard_kbase_platform_rk_init_opp_table -+#define kbase_platform_rk_shutdown midgard_kbase_platform_rk_shutdown -+#define kbase_pm_always_on_policy_ops midgard_kbase_pm_always_on_policy_ops -+#define kbase_pm_cache_snoop_disable midgard_kbase_pm_cache_snoop_disable -+#define kbase_pm_cache_snoop_enable midgard_kbase_pm_cache_snoop_enable -+#define kbase_pm_ca_get_core_mask midgard_kbase_pm_ca_get_core_mask -+#define kbase_pm_ca_init midgard_kbase_pm_ca_init -+#define kbase_pm_ca_term midgard_kbase_pm_ca_term -+#define kbase_pm_clock_off midgard_kbase_pm_clock_off -+#define kbase_pm_clock_on midgard_kbase_pm_clock_on -+#define kbase_pm_coarse_demand_policy_ops midgard_kbase_pm_coarse_demand_policy_ops -+#define kbase_pm_context_active_handle_suspend midgard_kbase_pm_context_active_handle_suspend -+#define kbase_pm_context_active midgard_kbase_pm_context_active -+#define kbase_pm_context_idle midgard_kbase_pm_context_idle -+#define kbase_pm_disable_interrupts midgard_kbase_pm_disable_interrupts -+#define kbase_pm_disable_interrupts_nolock midgard_kbase_pm_disable_interrupts_nolock -+#define kbase_pm_do_poweroff midgard_kbase_pm_do_poweroff -+#define kbase_pm_do_poweron midgard_kbase_pm_do_poweron -+#define kbasep_mem_profile_debugfs_insert midgard_kbasep_mem_profile_debugfs_insert -+#define kbasep_mem_profile_debugfs_remove midgard_kbasep_mem_profile_debugfs_remove -+#define kbase_pm_enable_interrupts midgard_kbase_pm_enable_interrupts -+#define kbase_pm_get_active_cores midgard_kbase_pm_get_active_cores -+#define kbase_pm_get_policy midgard_kbase_pm_get_policy -+#define kbase_pm_get_present_cores midgard_kbase_pm_get_present_cores -+#define kbase_pm_get_ready_cores midgard_kbase_pm_get_ready_cores -+#define kbase_pm_get_trans_cores midgard_kbase_pm_get_trans_cores -+#define kbase_pm_halt midgard_kbase_pm_halt -+#define kbase_pm_init_hw midgard_kbase_pm_init_hw -+#define kbase_pm_list_policies midgard_kbase_pm_list_policies -+#define kbase_pm_metrics_update midgard_kbase_pm_metrics_update -+#define kbase_pm_policy_init midgard_kbase_pm_policy_init -+#define kbase_pm_policy_term midgard_kbase_pm_policy_term -+#define kbase_pm_power_changed midgard_kbase_pm_power_changed -+#define kbase_pm_powerup midgard_kbase_pm_powerup -+#define kbase_pm_register_access_disable midgard_kbase_pm_register_access_disable -+#define kbase_pm_register_access_enable midgard_kbase_pm_register_access_enable -+#define kbase_pm_release_gpu_cycle_counter midgard_kbase_pm_release_gpu_cycle_counter -+#define kbase_pm_release_gpu_cycle_counter_nolock midgard_kbase_pm_release_gpu_cycle_counter_nolock -+#define kbase_pm_request_gpu_cycle_counter_l2_is_on midgard_kbase_pm_request_gpu_cycle_counter_l2_is_on -+#define kbase_pm_request_gpu_cycle_counter midgard_kbase_pm_request_gpu_cycle_counter -+#define kbase_pm_reset_done midgard_kbase_pm_reset_done -+#define kbase_pm_resume midgard_kbase_pm_resume -+#define kbase_pm_set_debug_core_mask midgard_kbase_pm_set_debug_core_mask -+#define kbase_pm_set_policy midgard_kbase_pm_set_policy -+#define kbase_pm_suspend midgard_kbase_pm_suspend -+#define kbase_pm_update_active midgard_kbase_pm_update_active -+#define kbase_pm_update_cores_state midgard_kbase_pm_update_cores_state -+#define kbase_pm_update_cores_state_nolock midgard_kbase_pm_update_cores_state_nolock -+#define kbase_pm_wait_for_poweroff_complete midgard_kbase_pm_wait_for_poweroff_complete -+#define kbasep_os_process_page_usage_update midgard_kbasep_os_process_page_usage_update -+#define kbasep_platform_device_init midgard_kbasep_platform_device_init -+#define kbasep_platform_device_term midgard_kbasep_platform_device_term -+#define kbasep_pm_metrics_init midgard_kbasep_pm_metrics_init -+#define kbasep_pm_metrics_term midgard_kbasep_pm_metrics_term -+#define kbasep_regs_history_debugfs_init midgard_kbasep_regs_history_debugfs_init -+#define kbasep_remove_waiting_soft_job midgard_kbasep_remove_waiting_soft_job -+#define kbase_prepare_soft_job midgard_kbase_prepare_soft_job -+#define kbase_prepare_to_reset_gpu_locked midgard_kbase_prepare_to_reset_gpu_locked -+#define kbase_prepare_to_reset_gpu midgard_kbase_prepare_to_reset_gpu -+#define kbase_process_soft_job midgard_kbase_process_soft_job -+#define kbasep_soft_job_timeout_worker midgard_kbasep_soft_job_timeout_worker -+#define kbase_region_tracker_find_region_base_address midgard_kbase_region_tracker_find_region_base_address -+#define kbase_region_tracker_find_region_enclosing_address midgard_kbase_region_tracker_find_region_enclosing_address -+#define kbase_region_tracker_init_jit midgard_kbase_region_tracker_init_jit -+#define kbase_region_tracker_init midgard_kbase_region_tracker_init -+#define kbase_region_tracker_term midgard_kbase_region_tracker_term -+#define kbase_reg_read midgard_kbase_reg_read -+#define kbase_reg_write midgard_kbase_reg_write -+#define kbase_release_device midgard_kbase_release_device -+#define kbase_release_interrupts midgard_kbase_release_interrupts -+#define kbase_reset_gpu_locked midgard_kbase_reset_gpu_locked -+#define kbase_reset_gpu midgard_kbase_reset_gpu -+#define kbase_reset_gpu_silent midgard_kbase_reset_gpu_silent -+#define kbase_resume_suspended_soft_jobs midgard_kbase_resume_suspended_soft_jobs -+#define kbase_scale_static_power midgard_kbase_scale_static_power -+#define kbase_set_custom_irq_handler midgard_kbase_set_custom_irq_handler -+#define kbase_simple_ipa_model_ops midgard_kbase_simple_ipa_model_ops -+#define kbase_soft_event_update midgard_kbase_soft_event_update -+#define kbase_soft_event_wait_callback midgard_kbase_soft_event_wait_callback -+#define kbase_sticky_resource_acquire midgard_kbase_sticky_resource_acquire -+#define kbase_sticky_resource_init midgard_kbase_sticky_resource_init -+#define kbase_sticky_resource_release midgard_kbase_sticky_resource_release -+#define kbase_sticky_resource_term midgard_kbase_sticky_resource_term -+#define kbase_sync_fence_in_cancel_wait midgard_kbase_sync_fence_in_cancel_wait -+#define kbase_sync_fence_in_dump midgard_kbase_sync_fence_in_dump -+#define kbase_sync_fence_in_from_fd midgard_kbase_sync_fence_in_from_fd -+#define kbase_sync_fence_in_info_get midgard_kbase_sync_fence_in_info_get -+#define kbase_sync_fence_in_remove midgard_kbase_sync_fence_in_remove -+#define kbase_sync_fence_in_wait midgard_kbase_sync_fence_in_wait -+#define kbase_sync_fence_out_create midgard_kbase_sync_fence_out_create -+#define kbase_sync_fence_out_info_get midgard_kbase_sync_fence_out_info_get -+#define kbase_sync_fence_out_remove midgard_kbase_sync_fence_out_remove -+#define kbase_sync_fence_out_trigger midgard_kbase_sync_fence_out_trigger -+#define kbase_sync_fence_stream_create midgard_kbase_sync_fence_stream_create -+#define kbase_sync_fence_validate midgard_kbase_sync_fence_validate -+#define kbase_sync_fence_wait_worker midgard_kbase_sync_fence_wait_worker -+#define kbase_synchronize_irqs midgard_kbase_synchronize_irqs -+#define kbase_sync_now midgard_kbase_sync_now -+#define kbase_sync_single_for_cpu midgard_kbase_sync_single_for_cpu -+#define kbase_sync_single_for_device midgard_kbase_sync_single_for_device -+#define kbase_sync_single midgard_kbase_sync_single -+#define kbase_sync_status_string midgard_kbase_sync_status_string -+#define kbase_timeline_name midgard_kbase_timeline_name -+#define __kbase_tlstream_aux_devfreq_target midgard___kbase_tlstream_aux_devfreq_target -+#define __kbase_tlstream_aux_pagefault midgard___kbase_tlstream_aux_pagefault -+#define __kbase_tlstream_aux_pagesalloc midgard___kbase_tlstream_aux_pagesalloc -+#define __kbase_tlstream_aux_pm_state midgard___kbase_tlstream_aux_pm_state -+#define __kbase_tlstream_aux_protected_enter_end midgard___kbase_tlstream_aux_protected_enter_end -+#define __kbase_tlstream_aux_protected_enter_start midgard___kbase_tlstream_aux_protected_enter_start -+#define __kbase_tlstream_aux_protected_leave_end midgard___kbase_tlstream_aux_protected_leave_end -+#define __kbase_tlstream_aux_protected_leave_start midgard___kbase_tlstream_aux_protected_leave_start -+#define kbase_tlstream_init midgard_kbase_tlstream_init -+#define __kbase_tlstream_jd_gpu_soft_reset midgard___kbase_tlstream_jd_gpu_soft_reset -+#define kbase_tlstream_term midgard_kbase_tlstream_term -+#define __kbase_tlstream_tl_attrib_as_config midgard___kbase_tlstream_tl_attrib_as_config -+#define __kbase_tlstream_tl_attrib_atom_config midgard___kbase_tlstream_tl_attrib_atom_config -+#define __kbase_tlstream_tl_attrib_atom_jit midgard___kbase_tlstream_tl_attrib_atom_jit -+#define __kbase_tlstream_tl_attrib_atom_priority midgard___kbase_tlstream_tl_attrib_atom_priority -+#define __kbase_tlstream_tl_attrib_atom_state midgard___kbase_tlstream_tl_attrib_atom_state -+#define __kbase_tlstream_tl_del_atom midgard___kbase_tlstream_tl_del_atom -+#define __kbase_tlstream_tl_del_ctx midgard___kbase_tlstream_tl_del_ctx -+#define __kbase_tlstream_tl_event_atom_softstop_ex midgard___kbase_tlstream_tl_event_atom_softstop_ex -+#define __kbase_tlstream_tl_event_atom_softstop_issue midgard___kbase_tlstream_tl_event_atom_softstop_issue -+#define __kbase_tlstream_tl_event_lpu_softstop midgard___kbase_tlstream_tl_event_lpu_softstop -+#define __kbase_tlstream_tl_new_atom midgard___kbase_tlstream_tl_new_atom -+#define __kbase_tlstream_tl_new_ctx midgard___kbase_tlstream_tl_new_ctx -+#define __kbase_tlstream_tl_nret_as_ctx midgard___kbase_tlstream_tl_nret_as_ctx -+#define __kbase_tlstream_tl_nret_atom_as midgard___kbase_tlstream_tl_nret_atom_as -+#define __kbase_tlstream_tl_nret_atom_ctx midgard___kbase_tlstream_tl_nret_atom_ctx -+#define __kbase_tlstream_tl_nret_atom_lpu midgard___kbase_tlstream_tl_nret_atom_lpu -+#define __kbase_tlstream_tl_nret_ctx_lpu midgard___kbase_tlstream_tl_nret_ctx_lpu -+#define __kbase_tlstream_tl_ret_as_ctx midgard___kbase_tlstream_tl_ret_as_ctx -+#define __kbase_tlstream_tl_ret_atom_as midgard___kbase_tlstream_tl_ret_atom_as -+#define __kbase_tlstream_tl_ret_atom_ctx midgard___kbase_tlstream_tl_ret_atom_ctx -+#define __kbase_tlstream_tl_ret_atom_lpu midgard___kbase_tlstream_tl_ret_atom_lpu -+#define __kbase_tlstream_tl_ret_ctx_lpu midgard___kbase_tlstream_tl_ret_ctx_lpu -+#define kbase_unmap_external_resource midgard_kbase_unmap_external_resource -+#define kbase_update_region_flags midgard_kbase_update_region_flags -+#define kbase_vinstr_hwcnt_reader_setup midgard_kbase_vinstr_hwcnt_reader_setup -+#define kbase_vinstr_init midgard_kbase_vinstr_init -+#define kbase_vinstr_resume midgard_kbase_vinstr_resume -+#define kbase_vinstr_suspend midgard_kbase_vinstr_suspend -+#define kbase_vinstr_term midgard_kbase_vinstr_term -+#define kbase_vmap midgard_kbase_vmap -+#define kbase_vmap_prot midgard_kbase_vmap_prot -+#define kbase_vm_ops midgard_kbase_vm_ops -+#define kbase_vunmap midgard_kbase_vunmap -+#define _mali_profiling_control midgard__mali_profiling_control -+#define platform_funcs midgard_platform_funcs -+#define pm_callbacks midgard_pm_callbacks -+#define rk_kbase_device_runtime_disable midgard_rk_kbase_device_runtime_disable -+#define rk_kbase_device_runtime_init midgard_rk_kbase_device_runtime_init -+#endif -diff --git a/drivers/gpu/arm/midgard/sconscript b/drivers/gpu/arm/midgard/sconscript -new file mode 100755 -index 000000000..ff23d7aeb ---- /dev/null -+++ b/drivers/gpu/arm/midgard/sconscript -@@ -0,0 +1,92 @@ -+# -+# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+import sys -+Import('env') -+ -+SConscript( 'tests/sconscript' ) -+ -+mock_test = 0 -+ -+# Fake platform is a transient solution for GPL drivers running in kernel that does not provide configuration via platform data. -+# For such kernels fake_platform_device should be set to 1. For kernels providing platform data fake_platform_device should be set to 0. -+if env['platform_config']=='devicetree' or env['platform_config']=='juno_soc': -+ fake_platform_device = 0 -+else: -+ fake_platform_device = 1 -+ -+# Source files required for kbase. -+kbase_src = [ -+ Glob('*.c'), -+ Glob('backend/*/*.c'), -+ Glob('internal/*/*.c'), -+ Glob('ipa/*.c') -+] -+ -+if env['platform_config']=='juno_soc': -+ kbase_src += [Glob('platform/devicetree/*.c')] -+else: -+ kbase_src += [Glob('platform/%s/*.c' % env['platform_config'])] -+ -+if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit'] == '1': -+ kbase_src += [Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock/*.c')] -+ mock_test = 1 -+ -+# we need platform config for GPL version using fake platform -+if fake_platform_device==1: -+ # Check if we are compiling for PBX -+ if env.KernelConfigEnabled("CONFIG_MACH_REALVIEW_PBX") and \ -+ env["platform_config"] in {"vexpress", "vexpress_6xvirtex7_10mhz"}: -+ sys.stderr.write("WARNING: Building for a PBX kernel but with platform_config=vexpress*\n") -+ # if the file platform config file is in the tpip directory then use that, otherwise use the default config directory -+ if Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])): -+ kbase_src += Glob('#kernel/drivers/gpu/arm/midgard/config/tpip/*%s.c' % (env['platform_config'])) -+ else: -+ kbase_src += Glob('#kernel/drivers/gpu/arm/midgard/config/*%s.c' % (env['platform_config'])) -+ -+make_args = env.kernel_get_config_defines(ret_list = True, -+ fake = fake_platform_device) + [ -+ 'PLATFORM=%s' % env['platform'], -+ 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], -+ 'MALI_KERNEL_TEST_API=%s' % env['debug'], -+ 'MALI_UNIT_TEST=%s' % env['unit'], -+ 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], -+ 'MALI_MOCK_TEST=%s' % mock_test, -+ 'MALI_CUSTOMER_RELEASE=%s' % env['release'], -+ 'MALI_INSTRUMENTATION_LEVEL=%s' % env['instr'], -+ 'MALI_COVERAGE=%s' % env['coverage'], -+ 'MALI_BUS_LOG=%s' % env['buslog'] -+] -+ -+kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, -+ make_args = make_args) -+ -+# Add a dependency on kds.ko. -+# Only necessary when KDS is not built into the kernel. -+# -+if env['os'] != 'android': -+ if not env.KernelConfigEnabled("CONFIG_KDS"): -+ env.Depends(kbase, '$STATIC_LIB_PATH/kds.ko') -+ -+# need Module.symvers from ump.ko build -+if int(env['ump']) == 1: -+ env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') -+ -+if 'smc_protected_mode_switcher' in env: -+ env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') -+ -+env.KernelObjTarget('kbase', kbase) -+ -+env.AppendUnique(BASE=['cutils_linked_list']) -diff --git a/drivers/gpu/arm/midgard/tests/Kbuild b/drivers/gpu/arm/midgard/tests/Kbuild -new file mode 100755 -index 000000000..b4bed0473 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/Kbuild -@@ -0,0 +1,17 @@ -+# -+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+obj-$(CONFIG_MALI_KUTF) += kutf/ -+obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ -diff --git a/drivers/gpu/arm/midgard/tests/Kconfig b/drivers/gpu/arm/midgard/tests/Kconfig -new file mode 100755 -index 000000000..da0515c06 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/Kconfig -@@ -0,0 +1,17 @@ -+# -+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" -+source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" -diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h -new file mode 100755 -index 000000000..0d145e42a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h -@@ -0,0 +1,65 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KERNEL_UTF_MEM_H_ -+#define _KERNEL_UTF_MEM_H_ -+ -+/* kutf_mem.h -+ * Functions for management of memory pools in the kernel. -+ * -+ * This module implements a memory pool allocator, allowing a test -+ * implementation to allocate linked allocations which can then be freed by a -+ * single free which releases all of the resources held by the entire pool. -+ * -+ * Note that it is not possible to free single resources within the pool once -+ * allocated. -+ */ -+ -+#include -+ -+/** -+ * struct kutf_mempool - the memory pool context management structure -+ * @head: list head on which the allocations in this context are added to -+ * -+ */ -+struct kutf_mempool { -+ struct list_head head; -+}; -+ -+/** -+ * kutf_mempool_init() - Initialize a memory pool. -+ * @pool: Memory pool structure to initialize, provided by the user -+ * -+ * Return: zero on success -+ */ -+int kutf_mempool_init(struct kutf_mempool *pool); -+ -+/** -+ * kutf_mempool_alloc() - Allocate memory from a pool -+ * @pool: Memory pool to allocate from -+ * @size: Size of memory wanted in number of bytes -+ * -+ * Return: Pointer to memory on success, NULL on failure. -+ */ -+void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size); -+ -+/** -+ * kutf_mempool_destroy() - Destroy a memory pool, freeing all memory within it. -+ * @pool: The memory pool to free -+ */ -+void kutf_mempool_destroy(struct kutf_mempool *pool); -+#endif /* _KERNEL_UTF_MEM_H_ */ -diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h -new file mode 100755 -index 000000000..1cc85f1b7 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h -@@ -0,0 +1,121 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KERNEL_UTF_RESULTSET_H_ -+#define _KERNEL_UTF_RESULTSET_H_ -+ -+/* kutf_resultset.h -+ * Functions and structures for handling test results and result sets. -+ * -+ * This section of the kernel UTF contains structures and functions used for the -+ * management of Results and Result Sets. -+ */ -+ -+/** -+ * enum kutf_result_status - Status values for a single Test error. -+ * @KUTF_RESULT_BENCHMARK: Result is a meta-result containing benchmark -+ * results. -+ * @KUTF_RESULT_SKIP: The test was skipped. -+ * @KUTF_RESULT_UNKNOWN: The test has an unknown result. -+ * @KUTF_RESULT_PASS: The test result passed. -+ * @KUTF_RESULT_DEBUG: The test result passed, but raised a debug -+ * message. -+ * @KUTF_RESULT_INFO: The test result passed, but raised -+ * an informative message. -+ * @KUTF_RESULT_WARN: The test result passed, but raised a warning -+ * message. -+ * @KUTF_RESULT_FAIL: The test result failed with a non-fatal error. -+ * @KUTF_RESULT_FATAL: The test result failed with a fatal error. -+ * @KUTF_RESULT_ABORT: The test result failed due to a non-UTF -+ * assertion failure. -+ * @KUTF_RESULT_COUNT: The current number of possible status messages. -+ */ -+enum kutf_result_status { -+ KUTF_RESULT_BENCHMARK = -3, -+ KUTF_RESULT_SKIP = -2, -+ KUTF_RESULT_UNKNOWN = -1, -+ -+ KUTF_RESULT_PASS = 0, -+ KUTF_RESULT_DEBUG = 1, -+ KUTF_RESULT_INFO = 2, -+ KUTF_RESULT_WARN = 3, -+ KUTF_RESULT_FAIL = 4, -+ KUTF_RESULT_FATAL = 5, -+ KUTF_RESULT_ABORT = 6, -+ -+ KUTF_RESULT_COUNT -+}; -+ -+/* The maximum size of a kutf_result_status result when -+ * converted to a string -+ */ -+#define KUTF_ERROR_MAX_NAME_SIZE 21 -+ -+#ifdef __KERNEL__ -+ -+#include -+ -+/** -+ * struct kutf_result - Represents a single test result. -+ * @node: Next result in the list of results. -+ * @status: The status summary (pass / warn / fail / etc). -+ * @message: A more verbose status message. -+ */ -+struct kutf_result { -+ struct list_head node; -+ enum kutf_result_status status; -+ const char *message; -+}; -+ -+/** -+ * kutf_create_result_set() - Create a new result set -+ * to which results can be added. -+ * -+ * Return: The created resultset. -+ */ -+struct kutf_result_set *kutf_create_result_set(void); -+ -+/** -+ * kutf_add_result() - Add a result to the end of an existing resultset. -+ * -+ * @mempool: The memory pool to allocate the result storage from. -+ * @set: The resultset to add the result to. -+ * @status: The result status to add. -+ * @message: The result message to add. -+ */ -+void kutf_add_result(struct kutf_mempool *mempool, struct kutf_result_set *set, -+ enum kutf_result_status status, const char *message); -+ -+/** -+ * kutf_remove_result() - Remove a result from the head of a resultset. -+ * @set: The resultset. -+ * -+ * Return: result or NULL if there are no further results in the resultset. -+ */ -+struct kutf_result *kutf_remove_result( -+ struct kutf_result_set *set); -+ -+/** -+ * kutf_destroy_result_set() - Free a previously created resultset. -+ * -+ * @results: The result set whose resources to free. -+ */ -+void kutf_destroy_result_set(struct kutf_result_set *results); -+ -+#endif /* __KERNEL__ */ -+ -+#endif /* _KERNEL_UTF_RESULTSET_H_ */ -diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h -new file mode 100755 -index 000000000..754c3adb1 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h -@@ -0,0 +1,508 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KERNEL_UTF_SUITE_H_ -+#define _KERNEL_UTF_SUITE_H_ -+ -+/* kutf_suite.h -+ * Functions for management of test suites. -+ * -+ * This collection of data structures, macros, and functions are used to -+ * create Test Suites, Tests within those Test Suites, and Fixture variants -+ * of each test. -+ */ -+ -+#include -+#include -+ -+/** -+ * Pseudo-flag indicating an absence of any specified test class. Note that -+ * tests should not be annotated with this constant as it is simply a zero -+ * value; tests without a more specific class must be marked with the flag -+ * KUTF_F_TEST_GENERIC. -+ */ -+#define KUTF_F_TEST_NONE ((unsigned int)(0)) -+ -+/** -+ * Class indicating this test is a smoke test. -+ * A given set of smoke tests should be quick to run, enabling rapid turn-around -+ * of "regress-on-commit" test runs. -+ */ -+#define KUTF_F_TEST_SMOKETEST ((unsigned int)(1 << 1)) -+ -+/** -+ * Class indicating this test is a performance test. -+ * These tests typically produce a performance metric, such as "time to run" or -+ * "frames per second", -+ */ -+#define KUTF_F_TEST_PERFORMANCE ((unsigned int)(1 << 2)) -+ -+/** -+ * Class indicating that this test is a deprecated test. -+ * These tests have typically been replaced by an alternative test which is -+ * more efficient, or has better coverage. -+ */ -+#define KUTF_F_TEST_DEPRECATED ((unsigned int)(1 << 3)) -+ -+/** -+ * Class indicating that this test is a known failure. -+ * These tests have typically been run and failed, but marking them as a known -+ * failure means it is easier to triage results. -+ * -+ * It is typically more convenient to triage known failures using the -+ * results database and web UI, as this means there is no need to modify the -+ * test code. -+ */ -+#define KUTF_F_TEST_EXPECTED_FAILURE ((unsigned int)(1 << 4)) -+ -+/** -+ * Class indicating that this test is a generic test, which is not a member of -+ * a more specific test class. Tests which are not created with a specific set -+ * of filter flags by the user are assigned this test class by default. -+ */ -+#define KUTF_F_TEST_GENERIC ((unsigned int)(1 << 5)) -+ -+/** -+ * Class indicating this test is a resource allocation failure test. -+ * A resource allocation failure test will test that an error code is -+ * correctly propagated when an allocation fails. -+ */ -+#define KUTF_F_TEST_RESFAIL ((unsigned int)(1 << 6)) -+ -+/** -+ * Additional flag indicating that this test is an expected failure when -+ * run in resource failure mode. These tests are never run when running -+ * the low resource mode. -+ */ -+#define KUTF_F_TEST_EXPECTED_FAILURE_RF ((unsigned int)(1 << 7)) -+ -+/** -+ * Flag reserved for user-defined filter zero. -+ */ -+#define KUTF_F_TEST_USER_0 ((unsigned int)(1 << 24)) -+ -+/** -+ * Flag reserved for user-defined filter one. -+ */ -+#define KUTF_F_TEST_USER_1 ((unsigned int)(1 << 25)) -+ -+/** -+ * Flag reserved for user-defined filter two. -+ */ -+#define KUTF_F_TEST_USER_2 ((unsigned int)(1 << 26)) -+ -+/** -+ * Flag reserved for user-defined filter three. -+ */ -+#define KUTF_F_TEST_USER_3 ((unsigned int)(1 << 27)) -+ -+/** -+ * Flag reserved for user-defined filter four. -+ */ -+#define KUTF_F_TEST_USER_4 ((unsigned int)(1 << 28)) -+ -+/** -+ * Flag reserved for user-defined filter five. -+ */ -+#define KUTF_F_TEST_USER_5 ((unsigned int)(1 << 29)) -+ -+/** -+ * Flag reserved for user-defined filter six. -+ */ -+#define KUTF_F_TEST_USER_6 ((unsigned int)(1 << 30)) -+ -+/** -+ * Flag reserved for user-defined filter seven. -+ */ -+#define KUTF_F_TEST_USER_7 ((unsigned int)(1 << 31)) -+ -+/** -+ * Pseudo-flag indicating that all test classes should be executed. -+ */ -+#define KUTF_F_TEST_ALL ((unsigned int)(0xFFFFFFFFU)) -+ -+/** -+ * union kutf_callback_data - Union used to store test callback data -+ * @ptr_value: pointer to the location where test callback data -+ * are stored -+ * @u32_value: a number which represents test callback data -+ */ -+union kutf_callback_data { -+ void *ptr_value; -+ u32 u32_value; -+}; -+ -+/** -+ * struct kutf_context - Structure representing a kernel test context -+ * @suite: Convenience pointer to the suite this context -+ * is running -+ * @test_fix: The fixture that is being run in this context -+ * @fixture_pool: The memory pool used for the duration of -+ * the fixture/text context. -+ * @fixture: The user provided fixture structure. -+ * @fixture_index: The index (id) of the current fixture. -+ * @fixture_name: The name of the current fixture (or NULL if unnamed). -+ * @test_data: Any user private data associated with this test -+ * @result_set: All the results logged by this test context -+ * @status: The status of the currently running fixture. -+ * @expected_status: The expected status on exist of the currently -+ * running fixture. -+ */ -+struct kutf_context { -+ struct kutf_suite *suite; -+ struct kutf_test_fixture *test_fix; -+ struct kutf_mempool fixture_pool; -+ void *fixture; -+ unsigned int fixture_index; -+ const char *fixture_name; -+ union kutf_callback_data test_data; -+ struct kutf_result_set *result_set; -+ enum kutf_result_status status; -+ enum kutf_result_status expected_status; -+}; -+ -+/** -+ * struct kutf_suite - Structure representing a kernel test suite -+ * @app: The application this suite belongs to. -+ * @name: The name of this suite. -+ * @suite_data: Any user private data associated with this -+ * suite. -+ * @create_fixture: Function used to create a new fixture instance -+ * @remove_fixture: Function used to destroy a new fixture instance -+ * @fixture_variants: The number of variants (must be at least 1). -+ * @suite_default_flags: Suite global filter flags which are set on -+ * all tests. -+ * @node: List node for suite_list -+ * @dir: The debugfs directory for this suite -+ * @test_list: List head to store all the tests which are -+ * part of this suite -+ */ -+struct kutf_suite { -+ struct kutf_application *app; -+ const char *name; -+ union kutf_callback_data suite_data; -+ void *(*create_fixture)(struct kutf_context *context); -+ void (*remove_fixture)(struct kutf_context *context); -+ unsigned int fixture_variants; -+ unsigned int suite_default_flags; -+ struct list_head node; -+ struct dentry *dir; -+ struct list_head test_list; -+}; -+ -+/* ============================================================================ -+ Application functions -+============================================================================ */ -+ -+/** -+ * kutf_create_application() - Create an in kernel test application. -+ * @name: The name of the test application. -+ * -+ * Return: pointer to the kutf_application on success or NULL -+ * on failure -+ */ -+struct kutf_application *kutf_create_application(const char *name); -+ -+/** -+ * kutf_destroy_application() - Destroy an in kernel test application. -+ * -+ * @app: The test application to destroy. -+ */ -+void kutf_destroy_application(struct kutf_application *app); -+ -+/* ============================================================================ -+ Suite functions -+============================================================================ */ -+ -+/** -+ * kutf_create_suite() - Create a kernel test suite. -+ * @app: The test application to create the suite in. -+ * @name: The name of the suite. -+ * @fixture_count: The number of fixtures to run over the test -+ * functions in this suite -+ * @create_fixture: Callback used to create a fixture. The returned value -+ * is stored in the fixture pointer in the context for -+ * use in the test functions. -+ * @remove_fixture: Callback used to remove a previously created fixture. -+ * -+ * Suite names must be unique. Should two suites with the same name be -+ * registered with the same application then this function will fail, if they -+ * are registered with different applications then the function will not detect -+ * this and the call will succeed. -+ * -+ * Return: pointer to the created kutf_suite on success or NULL -+ * on failure -+ */ -+struct kutf_suite *kutf_create_suite( -+ struct kutf_application *app, -+ const char *name, -+ unsigned int fixture_count, -+ void *(*create_fixture)(struct kutf_context *context), -+ void (*remove_fixture)(struct kutf_context *context)); -+ -+/** -+ * kutf_create_suite_with_filters() - Create a kernel test suite with user -+ * defined default filters. -+ * @app: The test application to create the suite in. -+ * @name: The name of the suite. -+ * @fixture_count: The number of fixtures to run over the test -+ * functions in this suite -+ * @create_fixture: Callback used to create a fixture. The returned value -+ * is stored in the fixture pointer in the context for -+ * use in the test functions. -+ * @remove_fixture: Callback used to remove a previously created fixture. -+ * @filters: Filters to apply to a test if it doesn't provide its own -+ * -+ * Suite names must be unique. Should two suites with the same name be -+ * registered with the same application then this function will fail, if they -+ * are registered with different applications then the function will not detect -+ * this and the call will succeed. -+ * -+ * Return: pointer to the created kutf_suite on success or NULL on failure -+ */ -+struct kutf_suite *kutf_create_suite_with_filters( -+ struct kutf_application *app, -+ const char *name, -+ unsigned int fixture_count, -+ void *(*create_fixture)(struct kutf_context *context), -+ void (*remove_fixture)(struct kutf_context *context), -+ unsigned int filters); -+ -+/** -+ * kutf_create_suite_with_filters_and_data() - Create a kernel test suite with -+ * user defined default filters. -+ * @app: The test application to create the suite in. -+ * @name: The name of the suite. -+ * @fixture_count: The number of fixtures to run over the test -+ * functions in this suite -+ * @create_fixture: Callback used to create a fixture. The returned value -+ * is stored in the fixture pointer in the context for -+ * use in the test functions. -+ * @remove_fixture: Callback used to remove a previously created fixture. -+ * @filters: Filters to apply to a test if it doesn't provide its own -+ * @suite_data: Suite specific callback data, provided during the -+ * running of the test in the kutf_context -+ * -+ * Return: pointer to the created kutf_suite on success or NULL -+ * on failure -+ */ -+struct kutf_suite *kutf_create_suite_with_filters_and_data( -+ struct kutf_application *app, -+ const char *name, -+ unsigned int fixture_count, -+ void *(*create_fixture)(struct kutf_context *context), -+ void (*remove_fixture)(struct kutf_context *context), -+ unsigned int filters, -+ union kutf_callback_data suite_data); -+ -+/** -+ * kutf_add_test() - Add a test to a kernel test suite. -+ * @suite: The suite to add the test to. -+ * @id: The ID of the test. -+ * @name: The name of the test. -+ * @execute: Callback to the test function to run. -+ * -+ * Note: As no filters are provided the test will use the suite filters instead -+ */ -+void kutf_add_test(struct kutf_suite *suite, -+ unsigned int id, -+ const char *name, -+ void (*execute)(struct kutf_context *context)); -+ -+/** -+ * kutf_add_test_with_filters() - Add a test to a kernel test suite with filters -+ * @suite: The suite to add the test to. -+ * @id: The ID of the test. -+ * @name: The name of the test. -+ * @execute: Callback to the test function to run. -+ * @filters: A set of filtering flags, assigning test categories. -+ */ -+void kutf_add_test_with_filters(struct kutf_suite *suite, -+ unsigned int id, -+ const char *name, -+ void (*execute)(struct kutf_context *context), -+ unsigned int filters); -+ -+/** -+ * kutf_add_test_with_filters_and_data() - Add a test to a kernel test suite -+ * with filters. -+ * @suite: The suite to add the test to. -+ * @id: The ID of the test. -+ * @name: The name of the test. -+ * @execute: Callback to the test function to run. -+ * @filters: A set of filtering flags, assigning test categories. -+ * @test_data: Test specific callback data, provoided during the -+ * running of the test in the kutf_context -+ */ -+void kutf_add_test_with_filters_and_data( -+ struct kutf_suite *suite, -+ unsigned int id, -+ const char *name, -+ void (*execute)(struct kutf_context *context), -+ unsigned int filters, -+ union kutf_callback_data test_data); -+ -+/* ============================================================================ -+ Test functions -+============================================================================ */ -+/** -+ * kutf_test_log_result_external() - Log a result which has been created -+ * externally into a in a standard form -+ * recognized by the log parser. -+ * @context: The test context the test is running in -+ * @message: The message for this result -+ * @new_status: The result status of this log message -+ */ -+void kutf_test_log_result_external( -+ struct kutf_context *context, -+ const char *message, -+ enum kutf_result_status new_status); -+ -+/** -+ * kutf_test_expect_abort() - Tell the kernel that you expect the current -+ * fixture to produce an abort. -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_expect_abort(struct kutf_context *context); -+ -+/** -+ * kutf_test_expect_fatal() - Tell the kernel that you expect the current -+ * fixture to produce a fatal error. -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_expect_fatal(struct kutf_context *context); -+ -+/** -+ * kutf_test_expect_fail() - Tell the kernel that you expect the current -+ * fixture to fail. -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_expect_fail(struct kutf_context *context); -+ -+/** -+ * kutf_test_expect_warn() - Tell the kernel that you expect the current -+ * fixture to produce a warning. -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_expect_warn(struct kutf_context *context); -+ -+/** -+ * kutf_test_expect_pass() - Tell the kernel that you expect the current -+ * fixture to pass. -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_expect_pass(struct kutf_context *context); -+ -+/** -+ * kutf_test_skip() - Tell the kernel that the test should be skipped. -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_skip(struct kutf_context *context); -+ -+/** -+ * kutf_test_skip_msg() - Tell the kernel that this test has been skipped, -+ * supplying a reason string. -+ * @context: The test context this test is running in. -+ * @message: A message string containing the reason for the skip. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a prebaked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_skip_msg(struct kutf_context *context, const char *message); -+ -+/** -+ * kutf_test_pass() - Tell the kernel that this test has passed. -+ * @context: The test context this test is running in. -+ * @message: A message string containing the reason for the pass. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a pre-baked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_pass(struct kutf_context *context, char const *message); -+ -+/** -+ * kutf_test_debug() - Send a debug message -+ * @context: The test context this test is running in. -+ * @message: A message string containing the debug information. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a pre-baked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_debug(struct kutf_context *context, char const *message); -+ -+/** -+ * kutf_test_info() - Send an information message -+ * @context: The test context this test is running in. -+ * @message: A message string containing the information message. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a pre-baked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_info(struct kutf_context *context, char const *message); -+ -+/** -+ * kutf_test_warn() - Send a warning message -+ * @context: The test context this test is running in. -+ * @message: A message string containing the warning message. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a pre-baked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_warn(struct kutf_context *context, char const *message); -+ -+/** -+ * kutf_test_fail() - Tell the kernel that a test has failed -+ * @context: The test context this test is running in. -+ * @message: A message string containing the failure message. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a pre-baked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_fail(struct kutf_context *context, char const *message); -+ -+/** -+ * kutf_test_fatal() - Tell the kernel that a test has triggered a fatal error -+ * @context: The test context this test is running in. -+ * @message: A message string containing the fatal error message. -+ * -+ * Note: The message must not be freed during the lifetime of the test run. -+ * This means it should either be a pre-baked string, or if a dynamic string -+ * is required it must be created with kutf_dsprintf which will store -+ * the resultant string in a buffer who's lifetime is the same as the test run. -+ */ -+void kutf_test_fatal(struct kutf_context *context, char const *message); -+ -+/** -+ * kutf_test_abort() - Tell the kernel that a test triggered an abort in the test -+ * -+ * @context: The test context this test is running in. -+ */ -+void kutf_test_abort(struct kutf_context *context); -+ -+#endif /* _KERNEL_UTF_SUITE_H_ */ -diff --git a/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h -new file mode 100755 -index 000000000..c458c1f73 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h -@@ -0,0 +1,55 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#ifndef _KERNEL_UTF_UTILS_H_ -+#define _KERNEL_UTF_UTILS_H_ -+ -+/* kutf_utils.h -+ * Utilities for the kernel UTF test infrastructure. -+ * -+ * This collection of library functions are provided for use by kernel UTF -+ * and users of kernel UTF which don't directly fit within the other -+ * code modules. -+ */ -+ -+#include -+ -+/** -+ * Maximum size of the message strings within kernel UTF, messages longer then -+ * this will be truncated. -+ */ -+#define KUTF_MAX_DSPRINTF_LEN 1024 -+ -+/** -+ * kutf_dsprintf() - dynamic sprintf -+ * @pool: memory pool to allocate from -+ * @fmt: The format string describing the string to document. -+ * @... The parameters to feed in to the format string. -+ * -+ * This function implements sprintf which dynamically allocates memory to store -+ * the string. The library will free the memory containing the string when the -+ * result set is cleared or destroyed. -+ * -+ * Note The returned string may be truncated to fit an internal temporary -+ * buffer, which is KUTF_MAX_DSPRINTF_LEN bytes in length. -+ * -+ * Return: Returns pointer to allocated string, or NULL on error. -+ */ -+const char *kutf_dsprintf(struct kutf_mempool *pool, -+ const char *fmt, ...); -+ -+#endif /* _KERNEL_UTF_UTILS_H_ */ -diff --git a/drivers/gpu/arm/midgard/tests/kutf/Kbuild b/drivers/gpu/arm/midgard/tests/kutf/Kbuild -new file mode 100755 -index 000000000..6b840c2ef ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/Kbuild -@@ -0,0 +1,20 @@ -+# -+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ccflags-y += -I$(src)/../include -+ -+obj-$(CONFIG_MALI_KUTF) += kutf.o -+ -+kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o -diff --git a/drivers/gpu/arm/midgard/tests/kutf/Kconfig b/drivers/gpu/arm/midgard/tests/kutf/Kconfig -new file mode 100755 -index 000000000..84364716a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/Kconfig -@@ -0,0 +1,22 @@ -+# -+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ -+config MALI_KUTF -+ tristate "Mali Kernel Unit Test Framework" -+ default n -+ help -+ Enables MALI testing framework. To compile it as a module, -+ choose M here - this will generate a single module called kutf. -diff --git a/drivers/gpu/arm/midgard/tests/kutf/Makefile b/drivers/gpu/arm/midgard/tests/kutf/Makefile -new file mode 100755 -index 000000000..010c92ca3 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/Makefile -@@ -0,0 +1,29 @@ -+# -+# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+# linux build system bootstrap for out-of-tree module -+ -+# default to building for the host -+ARCH ?= $(shell uname -m) -+ -+ifeq ($(KDIR),) -+$(error Must specify KDIR to point to the kernel to target)) -+endif -+ -+all: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules -+ -+clean: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c -new file mode 100755 -index 000000000..5408e57d4 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c -@@ -0,0 +1,94 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* Kernel UTF memory management functions */ -+ -+#include -+#include -+#include -+ -+#include -+ -+ -+/** -+ * struct kutf_alloc_entry - Structure representing an allocation. -+ * @node: List node for use with kutf_mempool. -+ * @data: Data area of the allocation -+ */ -+struct kutf_alloc_entry { -+ struct list_head node; -+ u8 data[0]; -+}; -+ -+int kutf_mempool_init(struct kutf_mempool *pool) -+{ -+ if (!pool) { -+ pr_err("NULL pointer passed to %s\n", __func__); -+ return -1; -+ } -+ -+ INIT_LIST_HEAD(&pool->head); -+ -+ return 0; -+} -+EXPORT_SYMBOL(kutf_mempool_init); -+ -+void kutf_mempool_destroy(struct kutf_mempool *pool) -+{ -+ struct list_head *remove; -+ struct list_head *tmp; -+ -+ if (!pool) { -+ pr_err("NULL pointer passed to %s\n", __func__); -+ return; -+ } -+ -+ list_for_each_safe(remove, tmp, &pool->head) { -+ struct kutf_alloc_entry *remove_alloc; -+ -+ remove_alloc = list_entry(remove, struct kutf_alloc_entry, node); -+ list_del(&remove_alloc->node); -+ kfree(remove_alloc); -+ } -+} -+EXPORT_SYMBOL(kutf_mempool_destroy); -+ -+void *kutf_mempool_alloc(struct kutf_mempool *pool, size_t size) -+{ -+ struct kutf_alloc_entry *ret; -+ -+ if (!pool) { -+ pr_err("NULL pointer passed to %s\n", __func__); -+ goto fail_pool; -+ } -+ -+ ret = kmalloc(sizeof(*ret) + size, GFP_KERNEL); -+ if (!ret) { -+ pr_err("Failed to allocate memory\n"); -+ goto fail_alloc; -+ } -+ -+ INIT_LIST_HEAD(&ret->node); -+ list_add(&ret->node, &pool->head); -+ -+ return &ret->data[0]; -+ -+fail_alloc: -+fail_pool: -+ return NULL; -+} -+EXPORT_SYMBOL(kutf_mempool_alloc); -diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c -new file mode 100755 -index 000000000..5bd04969f ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c -@@ -0,0 +1,95 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* Kernel UTF result management functions */ -+ -+#include -+#include -+#include -+ -+#include -+ -+/** -+ * struct kutf_result_set - Represents a set of results. -+ * @results: Pointer to the linked list where the results are stored. -+ */ -+struct kutf_result_set { -+ struct list_head results; -+}; -+ -+struct kutf_result_set *kutf_create_result_set(void) -+{ -+ struct kutf_result_set *set; -+ -+ set = kmalloc(sizeof(*set), GFP_KERNEL); -+ if (!set) { -+ pr_err("Failed to allocate resultset"); -+ goto fail_alloc; -+ } -+ -+ INIT_LIST_HEAD(&set->results); -+ -+ return set; -+ -+fail_alloc: -+ return NULL; -+} -+ -+void kutf_add_result(struct kutf_mempool *mempool, -+ struct kutf_result_set *set, -+ enum kutf_result_status status, -+ const char *message) -+{ -+ /* Create the new result */ -+ struct kutf_result *new_result; -+ -+ BUG_ON(set == NULL); -+ -+ new_result = kutf_mempool_alloc(mempool, sizeof(*new_result)); -+ if (!new_result) { -+ pr_err("Result allocation failed\n"); -+ return; -+ } -+ -+ INIT_LIST_HEAD(&new_result->node); -+ new_result->status = status; -+ new_result->message = message; -+ -+ list_add_tail(&new_result->node, &set->results); -+} -+ -+void kutf_destroy_result_set(struct kutf_result_set *set) -+{ -+ if (!list_empty(&set->results)) -+ pr_err("kutf_destroy_result_set: Unread results from test\n"); -+ -+ kfree(set); -+} -+ -+struct kutf_result *kutf_remove_result(struct kutf_result_set *set) -+{ -+ if (!list_empty(&set->results)) { -+ struct kutf_result *ret; -+ -+ ret = list_first_entry(&set->results, struct kutf_result, node); -+ list_del(&ret->node); -+ return ret; -+ } -+ -+ return NULL; -+} -+ -diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c -new file mode 100755 -index 000000000..a7cfd3be9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c -@@ -0,0 +1,1041 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* Kernel UTF suite, test and fixture management including user to kernel -+ * interaction */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include -+#include -+#include -+ -+#if defined(CONFIG_DEBUG_FS) -+ -+/** -+ * struct kutf_application - Structure which represents kutf application -+ * @name: The name of this test application. -+ * @dir: The debugfs directory for this test -+ * @suite_list: List head to store all the suites which are part of this -+ * application -+ */ -+struct kutf_application { -+ const char *name; -+ struct dentry *dir; -+ struct list_head suite_list; -+}; -+ -+/** -+ * struct kutf_test_function - Structure which represents kutf test function -+ * @suite: Back reference to the suite this test function -+ * belongs to -+ * @filters: Filters that apply to this test function -+ * @test_id: Test ID -+ * @execute: Function to run for this test -+ * @test_data: Static data for this test -+ * @node: List node for test_list -+ * @variant_list: List head to store all the variants which can run on -+ * this function -+ * @dir: debugfs directory for this test function -+ */ -+struct kutf_test_function { -+ struct kutf_suite *suite; -+ unsigned int filters; -+ unsigned int test_id; -+ void (*execute)(struct kutf_context *context); -+ union kutf_callback_data test_data; -+ struct list_head node; -+ struct list_head variant_list; -+ struct dentry *dir; -+}; -+ -+/** -+ * struct kutf_test_fixture - Structure which holds information on the kutf -+ * test fixture -+ * @test_func: Test function this fixture belongs to -+ * @fixture_index: Index of this fixture -+ * @node: List node for variant_list -+ * @dir: debugfs directory for this test fixture -+ */ -+struct kutf_test_fixture { -+ struct kutf_test_function *test_func; -+ unsigned int fixture_index; -+ struct list_head node; -+ struct dentry *dir; -+}; -+ -+struct dentry *base_dir; -+ -+/** -+ * struct kutf_convert_table - Structure which keeps test results -+ * @result_name: Status of the test result -+ * @result: Status value for a single test -+ */ -+struct kutf_convert_table { -+ char result_name[50]; -+ enum kutf_result_status result; -+}; -+ -+struct kutf_convert_table kutf_convert[] = { -+#define ADD_UTF_RESULT(_name) \ -+{ \ -+ #_name, \ -+ _name, \ -+}, -+ADD_UTF_RESULT(KUTF_RESULT_BENCHMARK) -+ADD_UTF_RESULT(KUTF_RESULT_SKIP) -+ADD_UTF_RESULT(KUTF_RESULT_UNKNOWN) -+ADD_UTF_RESULT(KUTF_RESULT_PASS) -+ADD_UTF_RESULT(KUTF_RESULT_DEBUG) -+ADD_UTF_RESULT(KUTF_RESULT_INFO) -+ADD_UTF_RESULT(KUTF_RESULT_WARN) -+ADD_UTF_RESULT(KUTF_RESULT_FAIL) -+ADD_UTF_RESULT(KUTF_RESULT_FATAL) -+ADD_UTF_RESULT(KUTF_RESULT_ABORT) -+}; -+ -+#define UTF_CONVERT_SIZE (ARRAY_SIZE(kutf_convert)) -+ -+/** -+ * kutf_create_context() - Create a test context in which a specific fixture -+ * of an application will be run and its results -+ * reported back to the user -+ * @test_fix: Test fixture to be run. -+ * -+ * Return: Returns the created test context on success or NULL on failure -+ */ -+static struct kutf_context *kutf_create_context( -+ struct kutf_test_fixture *test_fix); -+ -+/** -+ * kutf_destroy_context() - Destroy a previously created test context -+ * @context: Test context to destroy -+ */ -+static void kutf_destroy_context(struct kutf_context *context); -+ -+/** -+ * kutf_set_result() - Set the test result against the specified test context -+ * @context: Test context -+ * @status: Result status -+ */ -+static void kutf_set_result(struct kutf_context *context, -+ enum kutf_result_status status); -+ -+/** -+ * kutf_set_expected_result() - Set the expected test result for the specified -+ * test context -+ * @context: Test context -+ * @expected_status: Expected result status -+ */ -+static void kutf_set_expected_result(struct kutf_context *context, -+ enum kutf_result_status expected_status); -+ -+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) -+/* Pre 3.4.0 kernels don't have the simple_open helper */ -+ -+/** -+ * simple_open() - Helper for file opening which stores the inode private data -+ * into the file private data -+ * @inode: File entry representation -+ * @file: A specific opening of the file -+ * -+ * Return: always 0; if inode private data do not exist, the file will not -+ * be assigned private data -+ */ -+static int simple_open(struct inode *inode, struct file *file) -+{ -+ if (inode->i_private) -+ file->private_data = inode->i_private; -+ return 0; -+} -+#endif -+ -+/** -+ * kutf_result_to_string() - Converts a KUTF result into a string -+ * @result_str: Output result string -+ * @result: Result status to convert -+ * -+ * Return: 1 if test result was successfully converted to string, 0 otherwise -+ */ -+static int kutf_result_to_string(char **result_str, -+ enum kutf_result_status result) -+{ -+ int i; -+ int ret = 0; -+ -+ for (i = 0; i < UTF_CONVERT_SIZE; i++) { -+ if (result == kutf_convert[i].result) { -+ *result_str = kutf_convert[i].result_name; -+ ret = 1; -+ } -+ } -+ return ret; -+} -+ -+/** -+ * kutf_debugfs_const_string_read() - Simple debugfs read callback which -+ * returns a constant string -+ * @file: Opened file to read from -+ * @buf: User buffer to write the data into -+ * @len: Amount of data to read -+ * @ppos: Offset into file to read from -+ * -+ * Return: On success, the number of bytes read and offset @ppos advanced by -+ * this number; on error, negative value -+ */ -+static ssize_t kutf_debugfs_const_string_read(struct file *file, -+ char __user *buf, size_t len, loff_t *ppos) -+{ -+ char *str = file->private_data; -+ -+ return simple_read_from_buffer(buf, len, ppos, str, strlen(str)); -+} -+ -+static const struct file_operations kutf_debugfs_const_string_ops = { -+ .owner = THIS_MODULE, -+ .open = simple_open, -+ .read = kutf_debugfs_const_string_read, -+ .llseek = default_llseek, -+}; -+ -+/** -+ * kutf_add_explicit_result() - Check if an explicit result needs to be added -+ * @context: KUTF test context -+ */ -+static void kutf_add_explicit_result(struct kutf_context *context) -+{ -+ switch (context->expected_status) { -+ case KUTF_RESULT_UNKNOWN: -+ if (context->status == KUTF_RESULT_UNKNOWN) -+ kutf_test_pass(context, "(implicit pass)"); -+ break; -+ -+ case KUTF_RESULT_WARN: -+ if (context->status == KUTF_RESULT_WARN) -+ kutf_test_pass(context, -+ "Pass (expected warn occurred)"); -+ else if (context->status != KUTF_RESULT_SKIP) -+ kutf_test_fail(context, -+ "Fail (expected warn missing)"); -+ break; -+ -+ case KUTF_RESULT_FAIL: -+ if (context->status == KUTF_RESULT_FAIL) -+ kutf_test_pass(context, -+ "Pass (expected fail occurred)"); -+ else if (context->status != KUTF_RESULT_SKIP) { -+ /* Force the expected status so the fail gets logged */ -+ context->expected_status = KUTF_RESULT_PASS; -+ kutf_test_fail(context, -+ "Fail (expected fail missing)"); -+ } -+ break; -+ -+ case KUTF_RESULT_FATAL: -+ if (context->status == KUTF_RESULT_FATAL) -+ kutf_test_pass(context, -+ "Pass (expected fatal occurred)"); -+ else if (context->status != KUTF_RESULT_SKIP) -+ kutf_test_fail(context, -+ "Fail (expected fatal missing)"); -+ break; -+ -+ case KUTF_RESULT_ABORT: -+ if (context->status == KUTF_RESULT_ABORT) -+ kutf_test_pass(context, -+ "Pass (expected abort occurred)"); -+ else if (context->status != KUTF_RESULT_SKIP) -+ kutf_test_fail(context, -+ "Fail (expected abort missing)"); -+ break; -+ default: -+ break; -+ } -+} -+ -+/** -+ * kutf_debugfs_run_open() Debugfs open callback for the "run" entry. -+ * @inode: inode of the opened file -+ * @file: Opened file to read from -+ * -+ * This function retrieves the test fixture data that is associated with the -+ * opened file and works back to get the test, suite and application so -+ * it can then run the test that is associated with the file entry. -+ * -+ * Return: 0 on success -+ */ -+static int kutf_debugfs_run_open(struct inode *inode, struct file *file) -+{ -+ struct kutf_test_fixture *test_fix = inode->i_private; -+ struct kutf_test_function *test_func = test_fix->test_func; -+ struct kutf_suite *suite = test_func->suite; -+ struct kutf_context *test_context; -+ -+ test_context = kutf_create_context(test_fix); -+ if (!test_context) -+ return -ENODEV; -+ -+ file->private_data = test_context; -+ -+ /* -+ * Call the create fixture function if required before the -+ * fixture is run -+ */ -+ if (suite->create_fixture) -+ test_context->fixture = suite->create_fixture(test_context); -+ -+ /* Only run the test if the fixture was created (if required) */ -+ if ((suite->create_fixture && test_context->fixture) || -+ (!suite->create_fixture)) { -+ /* Run this fixture */ -+ test_func->execute(test_context); -+ -+ if (suite->remove_fixture) -+ suite->remove_fixture(test_context); -+ -+ kutf_add_explicit_result(test_context); -+ } -+ return 0; -+} -+ -+/** -+ * kutf_debugfs_run_read() - Debugfs read callback for the "run" entry. -+ * @file: Opened file to read from -+ * @buf: User buffer to write the data into -+ * @len: Amount of data to read -+ * @ppos: Offset into file to read from -+ * -+ * This function emits the results which where logged during the opening of -+ * the file kutf_debugfs_run_open. -+ * Results will be emitted one at a time, once all the results have been read -+ * 0 will be returned to indicate there is no more data. -+ * -+ * Return: Number of bytes read. -+ */ -+static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, -+ size_t len, loff_t *ppos) -+{ -+ struct kutf_context *test_context = file->private_data; -+ struct kutf_result *res; -+ unsigned long bytes_not_copied; -+ ssize_t bytes_copied = 0; -+ -+ /* Note: This code assumes a result is read completely */ -+ res = kutf_remove_result(test_context->result_set); -+ if (res) { -+ char *kutf_str_ptr = NULL; -+ unsigned int kutf_str_len = 0; -+ unsigned int message_len = 0; -+ char separator = ':'; -+ char terminator = '\n'; -+ -+ kutf_result_to_string(&kutf_str_ptr, res->status); -+ if (kutf_str_ptr) -+ kutf_str_len = strlen(kutf_str_ptr); -+ -+ if (res->message) -+ message_len = strlen(res->message); -+ -+ if ((kutf_str_len + 1 + message_len + 1) > len) { -+ pr_err("Not enough space in user buffer for a single result"); -+ return 0; -+ } -+ -+ /* First copy the result string */ -+ if (kutf_str_ptr) { -+ bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, -+ kutf_str_len); -+ bytes_copied += kutf_str_len - bytes_not_copied; -+ if (bytes_not_copied) -+ goto exit; -+ } -+ -+ /* Then the separator */ -+ bytes_not_copied = copy_to_user(&buf[bytes_copied], -+ &separator, 1); -+ bytes_copied += 1 - bytes_not_copied; -+ if (bytes_not_copied) -+ goto exit; -+ -+ /* Finally Next copy the result string */ -+ if (res->message) { -+ bytes_not_copied = copy_to_user(&buf[bytes_copied], -+ res->message, message_len); -+ bytes_copied += message_len - bytes_not_copied; -+ if (bytes_not_copied) -+ goto exit; -+ } -+ -+ /* Finally the terminator */ -+ bytes_not_copied = copy_to_user(&buf[bytes_copied], -+ &terminator, 1); -+ bytes_copied += 1 - bytes_not_copied; -+ } -+exit: -+ return bytes_copied; -+} -+ -+/** -+ * kutf_debugfs_run_release() - Debugfs release callback for the "run" entry. -+ * @inode: File entry representation -+ * @file: A specific opening of the file -+ * -+ * Release any resources that where created during the opening of the file -+ * -+ * Return: 0 on success -+ */ -+static int kutf_debugfs_run_release(struct inode *inode, struct file *file) -+{ -+ struct kutf_context *test_context = file->private_data; -+ -+ kutf_destroy_context(test_context); -+ return 0; -+} -+ -+static const struct file_operations kutf_debugfs_run_ops = { -+ .owner = THIS_MODULE, -+ .open = kutf_debugfs_run_open, -+ .read = kutf_debugfs_run_read, -+ .release = kutf_debugfs_run_release, -+ .llseek = default_llseek, -+}; -+ -+/** -+ * create_fixture_variant() - Creates a fixture variant for the specified -+ * test function and index and the debugfs entries -+ * that represent it. -+ * @test_func: Test function -+ * @fixture_index: Fixture index -+ * -+ * Return: 0 on success, negative value corresponding to error code in failure -+ */ -+static int create_fixture_variant(struct kutf_test_function *test_func, -+ unsigned int fixture_index) -+{ -+ struct kutf_test_fixture *test_fix; -+ char name[11]; /* Enough to print the MAX_UINT32 + the null terminator */ -+ struct dentry *tmp; -+ int err; -+ -+ test_fix = kmalloc(sizeof(*test_fix), GFP_KERNEL); -+ if (!test_fix) { -+ pr_err("Failed to create debugfs directory when adding fixture\n"); -+ err = -ENOMEM; -+ goto fail_alloc; -+ } -+ -+ test_fix->test_func = test_func; -+ test_fix->fixture_index = fixture_index; -+ -+ snprintf(name, sizeof(name), "%d", fixture_index); -+ test_fix->dir = debugfs_create_dir(name, test_func->dir); -+ if (!test_func->dir) { -+ pr_err("Failed to create debugfs directory when adding fixture\n"); -+ /* Might not be the right error, we don't get it passed back to us */ -+ err = -EEXIST; -+ goto fail_dir; -+ } -+ -+ tmp = debugfs_create_file("type", S_IROTH, test_fix->dir, "fixture\n", -+ &kutf_debugfs_const_string_ops); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"type\" when adding fixture\n"); -+ /* Might not be the right error, we don't get it passed back to us */ -+ err = -EEXIST; -+ goto fail_file; -+ } -+ -+ tmp = debugfs_create_file("run", S_IROTH, test_fix->dir, test_fix, -+ &kutf_debugfs_run_ops); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"run\" when adding fixture\n"); -+ /* Might not be the right error, we don't get it passed back to us */ -+ err = -EEXIST; -+ goto fail_file; -+ } -+ -+ list_add(&test_fix->node, &test_func->variant_list); -+ return 0; -+ -+fail_file: -+ debugfs_remove_recursive(test_fix->dir); -+fail_dir: -+ kfree(test_fix); -+fail_alloc: -+ return err; -+} -+ -+/** -+ * kutf_remove_test_variant() - Destroy a previously created fixture variant. -+ * @test_fix: Test fixture -+ */ -+static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) -+{ -+ debugfs_remove_recursive(test_fix->dir); -+ kfree(test_fix); -+} -+ -+void kutf_add_test_with_filters_and_data( -+ struct kutf_suite *suite, -+ unsigned int id, -+ const char *name, -+ void (*execute)(struct kutf_context *context), -+ unsigned int filters, -+ union kutf_callback_data test_data) -+{ -+ struct kutf_test_function *test_func; -+ struct dentry *tmp; -+ unsigned int i; -+ -+ test_func = kmalloc(sizeof(*test_func), GFP_KERNEL); -+ if (!test_func) { -+ pr_err("Failed to allocate memory when adding test %s\n", name); -+ goto fail_alloc; -+ } -+ -+ INIT_LIST_HEAD(&test_func->variant_list); -+ -+ test_func->dir = debugfs_create_dir(name, suite->dir); -+ if (!test_func->dir) { -+ pr_err("Failed to create debugfs directory when adding test %s\n", name); -+ goto fail_dir; -+ } -+ -+ tmp = debugfs_create_file("type", S_IROTH, test_func->dir, "test\n", -+ &kutf_debugfs_const_string_ops); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); -+ goto fail_file; -+ } -+ -+ test_func->filters = filters; -+ tmp = debugfs_create_x32("filters", S_IROTH, test_func->dir, -+ &test_func->filters); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"filters\" when adding test %s\n", name); -+ goto fail_file; -+ } -+ -+ test_func->test_id = id; -+ tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, -+ &test_func->test_id); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); -+ goto fail_file; -+ } -+ -+ for (i = 0; i < suite->fixture_variants; i++) { -+ if (create_fixture_variant(test_func, i)) { -+ pr_err("Failed to create fixture %d when adding test %s\n", i, name); -+ goto fail_file; -+ } -+ } -+ -+ test_func->suite = suite; -+ test_func->execute = execute; -+ test_func->test_data = test_data; -+ -+ list_add(&test_func->node, &suite->test_list); -+ return; -+ -+fail_file: -+ debugfs_remove_recursive(test_func->dir); -+fail_dir: -+ kfree(test_func); -+fail_alloc: -+ return; -+} -+EXPORT_SYMBOL(kutf_add_test_with_filters_and_data); -+ -+void kutf_add_test_with_filters( -+ struct kutf_suite *suite, -+ unsigned int id, -+ const char *name, -+ void (*execute)(struct kutf_context *context), -+ unsigned int filters) -+{ -+ union kutf_callback_data data; -+ -+ data.ptr_value = NULL; -+ -+ kutf_add_test_with_filters_and_data(suite, -+ id, -+ name, -+ execute, -+ suite->suite_default_flags, -+ data); -+} -+EXPORT_SYMBOL(kutf_add_test_with_filters); -+ -+void kutf_add_test(struct kutf_suite *suite, -+ unsigned int id, -+ const char *name, -+ void (*execute)(struct kutf_context *context)) -+{ -+ union kutf_callback_data data; -+ -+ data.ptr_value = NULL; -+ -+ kutf_add_test_with_filters_and_data(suite, -+ id, -+ name, -+ execute, -+ suite->suite_default_flags, -+ data); -+} -+EXPORT_SYMBOL(kutf_add_test); -+ -+/** -+ * kutf_remove_test(): Remove a previously added test function. -+ * @test_func: Test function -+ */ -+static void kutf_remove_test(struct kutf_test_function *test_func) -+{ -+ struct list_head *pos; -+ struct list_head *tmp; -+ -+ list_for_each_safe(pos, tmp, &test_func->variant_list) { -+ struct kutf_test_fixture *test_fix; -+ -+ test_fix = list_entry(pos, struct kutf_test_fixture, node); -+ kutf_remove_test_variant(test_fix); -+ } -+ -+ list_del(&test_func->node); -+ debugfs_remove_recursive(test_func->dir); -+ kfree(test_func); -+} -+ -+struct kutf_suite *kutf_create_suite_with_filters_and_data( -+ struct kutf_application *app, -+ const char *name, -+ unsigned int fixture_count, -+ void *(*create_fixture)(struct kutf_context *context), -+ void (*remove_fixture)(struct kutf_context *context), -+ unsigned int filters, -+ union kutf_callback_data suite_data) -+{ -+ struct kutf_suite *suite; -+ struct dentry *tmp; -+ -+ suite = kmalloc(sizeof(*suite), GFP_KERNEL); -+ if (!suite) { -+ pr_err("Failed to allocate memory when creating suite %s\n", name); -+ goto fail_kmalloc; -+ } -+ -+ suite->dir = debugfs_create_dir(name, app->dir); -+ if (!suite->dir) { -+ pr_err("Failed to create debugfs directory when adding test %s\n", name); -+ goto fail_debugfs; -+ } -+ -+ tmp = debugfs_create_file("type", S_IROTH, suite->dir, "suite\n", -+ &kutf_debugfs_const_string_ops); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"type\" when adding test %s\n", name); -+ goto fail_file; -+ } -+ -+ INIT_LIST_HEAD(&suite->test_list); -+ suite->app = app; -+ suite->name = name; -+ suite->fixture_variants = fixture_count; -+ suite->create_fixture = create_fixture; -+ suite->remove_fixture = remove_fixture; -+ suite->suite_default_flags = filters; -+ suite->suite_data = suite_data; -+ -+ list_add(&suite->node, &app->suite_list); -+ -+ return suite; -+ -+fail_file: -+ debugfs_remove_recursive(suite->dir); -+fail_debugfs: -+ kfree(suite); -+fail_kmalloc: -+ return NULL; -+} -+EXPORT_SYMBOL(kutf_create_suite_with_filters_and_data); -+ -+struct kutf_suite *kutf_create_suite_with_filters( -+ struct kutf_application *app, -+ const char *name, -+ unsigned int fixture_count, -+ void *(*create_fixture)(struct kutf_context *context), -+ void (*remove_fixture)(struct kutf_context *context), -+ unsigned int filters) -+{ -+ union kutf_callback_data data; -+ -+ data.ptr_value = NULL; -+ return kutf_create_suite_with_filters_and_data(app, -+ name, -+ fixture_count, -+ create_fixture, -+ remove_fixture, -+ filters, -+ data); -+} -+EXPORT_SYMBOL(kutf_create_suite_with_filters); -+ -+struct kutf_suite *kutf_create_suite( -+ struct kutf_application *app, -+ const char *name, -+ unsigned int fixture_count, -+ void *(*create_fixture)(struct kutf_context *context), -+ void (*remove_fixture)(struct kutf_context *context)) -+{ -+ union kutf_callback_data data; -+ -+ data.ptr_value = NULL; -+ return kutf_create_suite_with_filters_and_data(app, -+ name, -+ fixture_count, -+ create_fixture, -+ remove_fixture, -+ KUTF_F_TEST_GENERIC, -+ data); -+} -+EXPORT_SYMBOL(kutf_create_suite); -+ -+/** -+ * kutf_destroy_suite() - Destroy a previously added test suite. -+ * @suite: Test suite -+ */ -+static void kutf_destroy_suite(struct kutf_suite *suite) -+{ -+ struct list_head *pos; -+ struct list_head *tmp; -+ -+ list_for_each_safe(pos, tmp, &suite->test_list) { -+ struct kutf_test_function *test_func; -+ -+ test_func = list_entry(pos, struct kutf_test_function, node); -+ kutf_remove_test(test_func); -+ } -+ -+ list_del(&suite->node); -+ debugfs_remove_recursive(suite->dir); -+ kfree(suite); -+} -+ -+struct kutf_application *kutf_create_application(const char *name) -+{ -+ struct kutf_application *app; -+ struct dentry *tmp; -+ -+ app = kmalloc(sizeof(*app), GFP_KERNEL); -+ if (!app) { -+ pr_err("Failed to create allocate memory when creating application %s\n", name); -+ goto fail_kmalloc; -+ } -+ -+ app->dir = debugfs_create_dir(name, base_dir); -+ if (!app->dir) { -+ pr_err("Failed to create debugfs direcotry when creating application %s\n", name); -+ goto fail_debugfs; -+ } -+ -+ tmp = debugfs_create_file("type", S_IROTH, app->dir, "application\n", -+ &kutf_debugfs_const_string_ops); -+ if (!tmp) { -+ pr_err("Failed to create debugfs file \"type\" when creating application %s\n", name); -+ goto fail_file; -+ } -+ -+ INIT_LIST_HEAD(&app->suite_list); -+ app->name = name; -+ -+ return app; -+ -+fail_file: -+ debugfs_remove_recursive(app->dir); -+fail_debugfs: -+ kfree(app); -+fail_kmalloc: -+ return NULL; -+} -+EXPORT_SYMBOL(kutf_create_application); -+ -+void kutf_destroy_application(struct kutf_application *app) -+{ -+ struct list_head *pos; -+ struct list_head *tmp; -+ -+ list_for_each_safe(pos, tmp, &app->suite_list) { -+ struct kutf_suite *suite; -+ -+ suite = list_entry(pos, struct kutf_suite, node); -+ kutf_destroy_suite(suite); -+ } -+ -+ debugfs_remove_recursive(app->dir); -+ kfree(app); -+} -+EXPORT_SYMBOL(kutf_destroy_application); -+ -+static struct kutf_context *kutf_create_context( -+ struct kutf_test_fixture *test_fix) -+{ -+ struct kutf_context *new_context; -+ -+ new_context = kmalloc(sizeof(*new_context), GFP_KERNEL); -+ if (!new_context) { -+ pr_err("Failed to allocate test context"); -+ goto fail_alloc; -+ } -+ -+ new_context->result_set = kutf_create_result_set(); -+ if (!new_context->result_set) { -+ pr_err("Failed to create resultset"); -+ goto fail_result_set; -+ } -+ -+ new_context->test_fix = test_fix; -+ /* Save the pointer to the suite as the callbacks will require it */ -+ new_context->suite = test_fix->test_func->suite; -+ new_context->status = KUTF_RESULT_UNKNOWN; -+ new_context->expected_status = KUTF_RESULT_UNKNOWN; -+ -+ kutf_mempool_init(&new_context->fixture_pool); -+ new_context->fixture = NULL; -+ new_context->fixture_index = test_fix->fixture_index; -+ new_context->fixture_name = NULL; -+ new_context->test_data = test_fix->test_func->test_data; -+ -+ return new_context; -+ -+fail_result_set: -+ kfree(new_context); -+fail_alloc: -+ return NULL; -+} -+ -+static void kutf_destroy_context(struct kutf_context *context) -+{ -+ kutf_destroy_result_set(context->result_set); -+ kutf_mempool_destroy(&context->fixture_pool); -+ kfree(context); -+} -+ -+static void kutf_set_result(struct kutf_context *context, -+ enum kutf_result_status status) -+{ -+ context->status = status; -+} -+ -+static void kutf_set_expected_result(struct kutf_context *context, -+ enum kutf_result_status expected_status) -+{ -+ context->expected_status = expected_status; -+} -+ -+/** -+ * kutf_test_log_result() - Log a result for the specified test context -+ * @context: Test context -+ * @message: Result string -+ * @new_status: Result status -+ */ -+static void kutf_test_log_result( -+ struct kutf_context *context, -+ const char *message, -+ enum kutf_result_status new_status) -+{ -+ if (context->status < new_status) -+ context->status = new_status; -+ -+ if (context->expected_status != new_status) -+ kutf_add_result(&context->fixture_pool, context->result_set, -+ new_status, message); -+} -+ -+void kutf_test_log_result_external( -+ struct kutf_context *context, -+ const char *message, -+ enum kutf_result_status new_status) -+{ -+ kutf_test_log_result(context, message, new_status); -+} -+EXPORT_SYMBOL(kutf_test_log_result_external); -+ -+void kutf_test_expect_abort(struct kutf_context *context) -+{ -+ kutf_set_expected_result(context, KUTF_RESULT_ABORT); -+} -+EXPORT_SYMBOL(kutf_test_expect_abort); -+ -+void kutf_test_expect_fatal(struct kutf_context *context) -+{ -+ kutf_set_expected_result(context, KUTF_RESULT_FATAL); -+} -+EXPORT_SYMBOL(kutf_test_expect_fatal); -+ -+void kutf_test_expect_fail(struct kutf_context *context) -+{ -+ kutf_set_expected_result(context, KUTF_RESULT_FAIL); -+} -+EXPORT_SYMBOL(kutf_test_expect_fail); -+ -+void kutf_test_expect_warn(struct kutf_context *context) -+{ -+ kutf_set_expected_result(context, KUTF_RESULT_WARN); -+} -+EXPORT_SYMBOL(kutf_test_expect_warn); -+ -+void kutf_test_expect_pass(struct kutf_context *context) -+{ -+ kutf_set_expected_result(context, KUTF_RESULT_PASS); -+} -+EXPORT_SYMBOL(kutf_test_expect_pass); -+ -+void kutf_test_skip(struct kutf_context *context) -+{ -+ kutf_set_result(context, KUTF_RESULT_SKIP); -+ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); -+ -+ kutf_test_log_result(context, "Test skipped", KUTF_RESULT_SKIP); -+} -+EXPORT_SYMBOL(kutf_test_skip); -+ -+void kutf_test_skip_msg(struct kutf_context *context, const char *message) -+{ -+ kutf_set_result(context, KUTF_RESULT_SKIP); -+ kutf_set_expected_result(context, KUTF_RESULT_UNKNOWN); -+ -+ kutf_test_log_result(context, kutf_dsprintf(&context->fixture_pool, -+ "Test skipped: %s", message), KUTF_RESULT_SKIP); -+ kutf_test_log_result(context, "!!!Test skipped!!!", KUTF_RESULT_SKIP); -+} -+EXPORT_SYMBOL(kutf_test_skip_msg); -+ -+void kutf_test_debug(struct kutf_context *context, char const *message) -+{ -+ kutf_test_log_result(context, message, KUTF_RESULT_DEBUG); -+} -+EXPORT_SYMBOL(kutf_test_debug); -+ -+void kutf_test_pass(struct kutf_context *context, char const *message) -+{ -+ static const char explicit_message[] = "(explicit pass)"; -+ -+ if (!message) -+ message = explicit_message; -+ -+ kutf_test_log_result(context, message, KUTF_RESULT_PASS); -+} -+EXPORT_SYMBOL(kutf_test_pass); -+ -+void kutf_test_info(struct kutf_context *context, char const *message) -+{ -+ kutf_test_log_result(context, message, KUTF_RESULT_INFO); -+} -+EXPORT_SYMBOL(kutf_test_info); -+ -+void kutf_test_warn(struct kutf_context *context, char const *message) -+{ -+ kutf_test_log_result(context, message, KUTF_RESULT_WARN); -+} -+EXPORT_SYMBOL(kutf_test_warn); -+ -+void kutf_test_fail(struct kutf_context *context, char const *message) -+{ -+ kutf_test_log_result(context, message, KUTF_RESULT_FAIL); -+} -+EXPORT_SYMBOL(kutf_test_fail); -+ -+void kutf_test_fatal(struct kutf_context *context, char const *message) -+{ -+ kutf_test_log_result(context, message, KUTF_RESULT_FATAL); -+} -+EXPORT_SYMBOL(kutf_test_fatal); -+ -+void kutf_test_abort(struct kutf_context *context) -+{ -+ kutf_test_log_result(context, "", KUTF_RESULT_ABORT); -+} -+EXPORT_SYMBOL(kutf_test_abort); -+ -+/** -+ * init_kutf_core() - Module entry point. -+ * -+ * Create the base entry point in debugfs. -+ */ -+static int __init init_kutf_core(void) -+{ -+ int ret; -+ -+ base_dir = debugfs_create_dir("kutf_tests", NULL); -+ if (!base_dir) { -+ ret = -ENODEV; -+ goto exit_dir; -+ } -+ -+ return 0; -+ -+exit_dir: -+ return ret; -+} -+ -+/** -+ * exit_kutf_core() - Module exit point. -+ * -+ * Remove the base entry point in debugfs. -+ */ -+static void __exit exit_kutf_core(void) -+{ -+ debugfs_remove_recursive(base_dir); -+} -+ -+#else /* defined(CONFIG_DEBUG_FS) */ -+ -+/** -+ * init_kutf_core() - Module entry point. -+ * -+ * Stub for when build against a kernel without debugfs support -+ */ -+static int __init init_kutf_core(void) -+{ -+ pr_debug("KUTF requires a kernel with debug fs support"); -+ -+ return -ENODEV; -+} -+ -+/** -+ * exit_kutf_core() - Module exit point. -+ * -+ * Stub for when build against a kernel without debugfs support -+ */ -+static void __exit exit_kutf_core(void) -+{ -+} -+#endif /* defined(CONFIG_DEBUG_FS) */ -+ -+MODULE_LICENSE("GPL"); -+ -+module_init(init_kutf_core); -+module_exit(exit_kutf_core); -diff --git a/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c b/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c -new file mode 100755 -index 000000000..a429a2dbf ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c -@@ -0,0 +1,71 @@ -+/* -+ * -+ * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+/* Kernel UTF utility functions */ -+ -+#include -+#include -+#include -+#include -+ -+#include -+#include -+ -+static char tmp_buffer[KUTF_MAX_DSPRINTF_LEN]; -+ -+DEFINE_MUTEX(buffer_lock); -+ -+const char *kutf_dsprintf(struct kutf_mempool *pool, -+ const char *fmt, ...) -+{ -+ va_list args; -+ int len; -+ int size; -+ void *buffer; -+ -+ mutex_lock(&buffer_lock); -+ va_start(args, fmt); -+ len = vsnprintf(tmp_buffer, sizeof(tmp_buffer), fmt, args); -+ va_end(args); -+ -+ if (len < 0) { -+ pr_err("kutf_dsprintf: Bad format dsprintf format %s\n", fmt); -+ goto fail_format; -+ } -+ -+ if (len >= sizeof(tmp_buffer)) { -+ pr_warn("kutf_dsprintf: Truncated dsprintf message %s\n", fmt); -+ size = sizeof(tmp_buffer); -+ } else { -+ size = len + 1; -+ } -+ -+ buffer = kutf_mempool_alloc(pool, size); -+ if (!buffer) -+ goto fail_alloc; -+ -+ memcpy(buffer, tmp_buffer, size); -+ mutex_unlock(&buffer_lock); -+ -+ return buffer; -+ -+fail_alloc: -+fail_format: -+ mutex_unlock(&buffer_lock); -+ return NULL; -+} -+EXPORT_SYMBOL(kutf_dsprintf); -diff --git a/drivers/gpu/arm/midgard/tests/kutf/sconscript b/drivers/gpu/arm/midgard/tests/kutf/sconscript -new file mode 100755 -index 000000000..d7f112448 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/kutf/sconscript -@@ -0,0 +1,21 @@ -+# -+# (C) COPYRIGHT 2014-2016, 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+Import('kutf_env') -+ -+make_args = kutf_env.kernel_get_config_defines(ret_list = True) -+ -+mod = kutf_env.BuildKernelModule('$STATIC_LIB_PATH/kutf.ko', Glob('*.c'), make_args = make_args) -+kutf_env.KernelObjTarget('kutf', mod) -diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild -new file mode 100755 -index 000000000..0cd9cebe9 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild -@@ -0,0 +1,20 @@ -+# -+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android -+ -+obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o -+ -+mali_kutf_irq_test-y := mali_kutf_irq_test_main.o -diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig -new file mode 100755 -index 000000000..16f68d15c ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig -@@ -0,0 +1,23 @@ -+# -+# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+config MALI_IRQ_LATENCY -+ tristate "Mali GPU IRQ latency measurement" -+ depends on MALI_MIDGARD && MALI_DEBUG && MALI_KUTF -+ default n -+ help -+ This option will build a test module mali_kutf_irq_test that -+ can determine the latency of the Mali GPU IRQ on your system. -+ Choosing M here will generate a single module called mali_kutf_irq_test. -diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile -new file mode 100755 -index 000000000..4e948767a ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile -@@ -0,0 +1,51 @@ -+# -+# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+# linux build system bootstrap for out-of-tree module -+ -+# default to building for the host -+ARCH ?= $(shell uname -m) -+ -+ifeq ($(KDIR),) -+$(error Must specify KDIR to point to the kernel to target)) -+endif -+ -+TEST_CCFLAGS := \ -+ -DMALI_DEBUG=$(MALI_DEBUG) \ -+ -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ -+ -DMALI_MODEL=$(MALI_MODEL) \ -+ -DMALI_NO_MALI=$(MALI_NO_MALI) \ -+ -DMALI_BASE_QA_LEAK=$(MALI_BASE_QA_LEAK) \ -+ -DMALI_BASE_QA_RESFAIL=$(MALI_BASE_QA_RESFAIL) \ -+ -DMALI_BASE_QA_USE_AFTER_FREE=$(MALI_BASE_QA_USE_AFTER_FREE) \ -+ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -+ -DMALI_USE_UMP=$(MALI_USE_UMP) \ -+ -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ -+ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ -+ $(SCONS_CFLAGS) \ -+ -I$(CURDIR)/../include \ -+ -I$(CURDIR)/../../../../../../include \ -+ -I$(CURDIR)/../../../ \ -+ -I$(CURDIR)/../../ \ -+ -I$(CURDIR)/../../backend/gpu \ -+ -I$(CURDIR)/ \ -+ -I$(srctree)/drivers/staging/android \ -+ -I$(srctree)/include/linux -+ -+all: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules -+ -+clean: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c -new file mode 100755 -index 000000000..e2ff4432b ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c -@@ -0,0 +1,257 @@ -+/* -+ * -+ * (C) COPYRIGHT 2016, 2017 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program, and can also be obtained -+ * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+ * Boston, MA 02110-1301, USA. -+ * -+ */ -+ -+ -+ -+#include -+#include -+#include -+ -+#include "mali_kbase.h" -+#include -+ -+#include -+#include -+ -+/* -+ * This file contains the code which is used for measuring interrupt latency -+ * of the Mali GPU IRQ. In particular, function mali_kutf_irq_latency() is -+ * used with this purpose and it is called within KUTF framework - a kernel -+ * unit test framework. The measured latency provided by this test should -+ * be representative for the latency of the Mali JOB/MMU IRQs as well. -+ */ -+ -+/* KUTF test application pointer for this test */ -+struct kutf_application *irq_app; -+ -+/** -+ * struct kutf_irq_fixture data - test fixture used by the test functions. -+ * @kbdev: kbase device for the GPU. -+ * -+ */ -+struct kutf_irq_fixture_data { -+ struct kbase_device *kbdev; -+}; -+ -+#define SEC_TO_NANO(s) ((s)*1000000000LL) -+ -+/* ID for the GPU IRQ */ -+#define GPU_IRQ_HANDLER 2 -+ -+#define NR_TEST_IRQS 1000000 -+ -+/* IRQ for the test to trigger. Currently MULTIPLE_GPU_FAULTS as we would not -+ * expect to see this in normal use (e.g., when Android is running). */ -+#define TEST_IRQ MULTIPLE_GPU_FAULTS -+ -+#define IRQ_TIMEOUT HZ -+ -+/* Kernel API for setting irq throttle hook callback and irq time in us*/ -+extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, -+ irq_handler_t custom_handler, -+ int irq_type); -+extern irqreturn_t kbase_gpu_irq_handler(int irq, void *data); -+ -+static DECLARE_WAIT_QUEUE_HEAD(wait); -+static bool triggered; -+static u64 irq_time; -+ -+static void *kbase_untag(void *ptr) -+{ -+ return (void *)(((uintptr_t) ptr) & ~3); -+} -+ -+/** -+ * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler -+ * @irq: IRQ number -+ * @data: Data associated with this IRQ -+ * -+ * Return: state of the IRQ -+ */ -+static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) -+{ -+ struct kbase_device *kbdev = kbase_untag(data); -+ u32 val; -+ -+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_STATUS), NULL); -+ if (val & TEST_IRQ) { -+ struct timespec64 tval; -+ -+ ktime_get_real_ts64(&tval); -+ irq_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); -+ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val, -+ NULL); -+ -+ triggered = true; -+ wake_up(&wait); -+ -+ return IRQ_HANDLED; -+ } -+ -+ /* Trigger main irq handler */ -+ return kbase_gpu_irq_handler(irq, data); -+} -+ -+/** -+ * mali_kutf_irq_default_create_fixture() - Creates the fixture data required -+ * for all the tests in the irq suite. -+ * @context: KUTF context. -+ * -+ * Return: Fixture data created on success or NULL on failure -+ */ -+static void *mali_kutf_irq_default_create_fixture( -+ struct kutf_context *context) -+{ -+ struct kutf_irq_fixture_data *data; -+ -+ data = kutf_mempool_alloc(&context->fixture_pool, -+ sizeof(struct kutf_irq_fixture_data)); -+ -+ if (!data) -+ goto fail; -+ -+ /* Acquire the kbase device */ -+ data->kbdev = kbase_find_device(-1); -+ if (data->kbdev == NULL) { -+ kutf_test_fail(context, "Failed to find kbase device"); -+ goto fail; -+ } -+ -+ return data; -+ -+fail: -+ return NULL; -+} -+ -+/** -+ * mali_kutf_irq_default_remove_fixture() - Destroy fixture data previously -+ * created by mali_kutf_irq_default_create_fixture. -+ * -+ * @context: KUTF context. -+ */ -+static void mali_kutf_irq_default_remove_fixture( -+ struct kutf_context *context) -+{ -+ struct kutf_irq_fixture_data *data = context->fixture; -+ struct kbase_device *kbdev = data->kbdev; -+ -+ kbase_release_device(kbdev); -+} -+ -+/** -+ * mali_kutf_irq_latency() - measure GPU IRQ latency -+ * @context: kutf context within which to perform the test -+ * -+ * The test triggers IRQs manually, and measures the -+ * time between triggering the IRQ and the IRQ handler being executed. -+ * -+ * This is not a traditional test, in that the pass/fail status has little -+ * meaning (other than indicating that the IRQ handler executed at all). Instead -+ * the results are in the latencies provided with the test result. There is no -+ * meaningful pass/fail result that can be obtained here, instead the latencies -+ * are provided for manual analysis only. -+ */ -+static void mali_kutf_irq_latency(struct kutf_context *context) -+{ -+ struct kutf_irq_fixture_data *data = context->fixture; -+ struct kbase_device *kbdev = data->kbdev; -+ u64 min_time = U64_MAX, max_time = 0, average_time = 0; -+ int i; -+ bool test_failed = false; -+ -+ /* Force GPU to be powered */ -+ kbase_pm_context_active(kbdev); -+ -+ kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, -+ GPU_IRQ_HANDLER); -+ -+ for (i = 0; i < NR_TEST_IRQS; i++) { -+ struct timespec64 tval; -+ u64 start_time; -+ int ret; -+ -+ triggered = false; -+ ktime_get_real_ts64(&tval); -+ start_time = SEC_TO_NANO(tval.tv_sec) + (tval.tv_nsec); -+ -+ /* Trigger fake IRQ */ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), -+ TEST_IRQ, NULL); -+ -+ ret = wait_event_timeout(wait, triggered != false, IRQ_TIMEOUT); -+ -+ if (ret == 0) { -+ kutf_test_fail(context, "Timed out waiting for IRQ\n"); -+ test_failed = true; -+ break; -+ } -+ -+ if ((irq_time - start_time) < min_time) -+ min_time = irq_time - start_time; -+ if ((irq_time - start_time) > max_time) -+ max_time = irq_time - start_time; -+ average_time += irq_time - start_time; -+ -+ udelay(10); -+ } -+ -+ /* Go back to default handler */ -+ kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); -+ -+ kbase_pm_context_idle(kbdev); -+ -+ if (!test_failed) { -+ const char *results; -+ -+ do_div(average_time, NR_TEST_IRQS); -+ results = kutf_dsprintf(&context->fixture_pool, -+ "Min latency = %lldns, Max latency = %lldns, Average latency = %lldns\n", -+ min_time, max_time, average_time); -+ kutf_test_pass(context, results); -+ } -+} -+ -+/** -+ * Module entry point for this test. -+ */ -+int mali_kutf_irq_test_main_init(void) -+{ -+ struct kutf_suite *suite; -+ -+ irq_app = kutf_create_application("irq"); -+ suite = kutf_create_suite(irq_app, "irq_default", -+ 1, mali_kutf_irq_default_create_fixture, -+ mali_kutf_irq_default_remove_fixture); -+ -+ kutf_add_test(suite, 0x0, "irq_latency", -+ mali_kutf_irq_latency); -+ return 0; -+} -+ -+/** -+ * Module exit point for this test. -+ */ -+void mali_kutf_irq_test_main_exit(void) -+{ -+ kutf_destroy_application(irq_app); -+} -+ -+module_init(mali_kutf_irq_test_main_init); -+module_exit(mali_kutf_irq_test_main_exit); -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("ARM Ltd."); -+MODULE_VERSION("1.0"); -diff --git a/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript -new file mode 100755 -index 000000000..ec837f164 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/sconscript -@@ -0,0 +1,30 @@ -+# -+# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+import os -+Import('env') -+ -+src = [Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/*.c'), Glob('#kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile')] -+ -+if env.GetOption('clean') : -+ env.Execute(Action("make clean", '[CLEAN] mali_kutf_irq_test')) -+ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) -+ env.KernelObjTarget('mali_kutf_irq_test', cmd) -+else: -+ makeAction=Action("cd ${SOURCE.dir} && make MALI_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_MODEL=${mali_model} MALI_NO_MALI=${no_mali} MALI_HW_VERSION=${hwver} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % (env.base_get_qa_settings(), env.kernel_get_config_defines()), '$MAKECOMSTR') -+ cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) -+ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') -+ env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') -+ env.KernelObjTarget('mali_kutf_irq_test', cmd) -diff --git a/drivers/gpu/arm/midgard/tests/sconscript b/drivers/gpu/arm/midgard/tests/sconscript -new file mode 100755 -index 000000000..5337e1078 ---- /dev/null -+++ b/drivers/gpu/arm/midgard/tests/sconscript -@@ -0,0 +1,37 @@ -+# -+# (C) COPYRIGHT 2010-2011, 2013, 2017 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# A copy of the licence is included with the program, and can also be obtained -+# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -+# Boston, MA 02110-1301, USA. -+# -+# -+ -+ -+Import ('env') -+ -+kutf_env = env.Clone() -+kutf_env.Append(CPPPATH = '#kernel/drivers/gpu/arm/midgard/tests/include') -+Export('kutf_env') -+ -+if Glob('internal/sconscript'): -+ SConscript('internal/sconscript') -+ -+if kutf_env['debug'] == '1': -+ SConscript('kutf/sconscript') -+ SConscript('mali_kutf_irq_test/sconscript') -+ -+ if Glob('kutf_test/sconscript'): -+ SConscript('kutf_test/sconscript') -+ -+ if Glob('kutf_test_runner/sconscript'): -+ SConscript('kutf_test_runner/sconscript') -+ -+if env['unit'] == '1': -+ SConscript('mali_kutf_ipa_test/sconscript') -+ SConscript('mali_kutf_vinstr_test/sconscript') -diff --git a/drivers/gpu/arm/sconscript b/drivers/gpu/arm/sconscript -new file mode 100755 -index 000000000..a06092bd5 ---- /dev/null -+++ b/drivers/gpu/arm/sconscript -@@ -0,0 +1,25 @@ -+# -+# (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU licence. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# SPDX-License-Identifier: GPL-2.0 -+# -+# -+ -+import glob -+ -+ -+SConscript('midgard/sconscript') -diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig -index ca868271f..2a9184156 100644 ---- a/drivers/gpu/drm/Kconfig -+++ b/drivers/gpu/drm/Kconfig -@@ -31,6 +31,10 @@ config DRM_MIPI_DBI - tristate - depends on DRM - -+config DRM_IGNORE_IOTCL_PERMIT -+ bool "Ignore drm ioctl permission" -+ depends on DRM && ANDROID && NO_GKI -+ - config DRM_MIPI_DSI - bool - depends on DRM -diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c -index aa1bb8629..4a1e90bdd 100644 ---- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c -+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c -@@ -13,6 +13,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -40,6 +41,20 @@ struct bridge_init { - struct device_node *node; - }; - -+static bool analogix_dp_bandwidth_ok(struct analogix_dp_device *dp, -+ const struct drm_display_mode *mode, -+ unsigned int rate, unsigned int lanes) -+{ -+ u32 max_bw, req_bw, bpp = 24; -+ -+ req_bw = mode->clock * bpp / 8; -+ max_bw = lanes * rate; -+ if (req_bw > max_bw) -+ return false; -+ -+ return true; -+} -+ - static int analogix_dp_init_dp(struct analogix_dp_device *dp) - { - int ret; -@@ -64,6 +79,46 @@ static int analogix_dp_init_dp(struct analogix_dp_device *dp) - return 0; - } - -+static int analogix_dp_panel_prepare(struct analogix_dp_device *dp) -+{ -+ int ret; -+ -+ mutex_lock(&dp->panel_lock); -+ -+ if (dp->panel_is_prepared) -+ goto out; -+ -+ ret = drm_panel_prepare(dp->plat_data->panel); -+ if (ret) -+ goto out; -+ -+ dp->panel_is_prepared = true; -+ -+out: -+ mutex_unlock(&dp->panel_lock); -+ return 0; -+} -+ -+static int analogix_dp_panel_unprepare(struct analogix_dp_device *dp) -+{ -+ int ret; -+ -+ mutex_lock(&dp->panel_lock); -+ -+ if (!dp->panel_is_prepared) -+ goto out; -+ -+ ret = drm_panel_unprepare(dp->plat_data->panel); -+ if (ret) -+ goto out; -+ -+ dp->panel_is_prepared = false; -+ -+out: -+ mutex_unlock(&dp->panel_lock); -+ return 0; -+} -+ - static int analogix_dp_detect_hpd(struct analogix_dp_device *dp) - { - int timeout_loop = 0; -@@ -108,6 +163,9 @@ static bool analogix_dp_detect_sink_psr(struct analogix_dp_device *dp) - unsigned char psr_version; - int ret; - -+ if (!device_property_read_bool(dp->dev, "support-psr")) -+ return 0; -+ - ret = drm_dp_dpcd_readb(&dp->aux, DP_PSR_SUPPORT, &psr_version); - if (ret != 1) { - dev_err(dp->dev, "failed to get PSR version, disable it\n"); -@@ -216,8 +274,24 @@ static int analogix_dp_set_enhanced_mode(struct analogix_dp_device *dp) - if (ret < 0) - return ret; - -+ if (!data) { -+ /* -+ * A setting of 1 indicates that this is an eDP device that -+ * uses only Enhanced Framing, independently of the setting by -+ * the source of ENHANCED_FRAME_EN -+ */ -+ ret = drm_dp_dpcd_readb(&dp->aux, DP_EDP_CONFIGURATION_CAP, -+ &data); -+ if (ret < 0) -+ return ret; -+ -+ data = !!(data & DP_FRAMING_CHANGE_CAP); -+ } -+ - analogix_dp_enable_enhanced_mode(dp, data); - -+ dp->link_train.enhanced_framing = data; -+ - return 0; - } - -@@ -233,32 +307,10 @@ static int analogix_dp_training_pattern_dis(struct analogix_dp_device *dp) - return ret < 0 ? ret : 0; - } - --static void --analogix_dp_set_lane_lane_pre_emphasis(struct analogix_dp_device *dp, -- int pre_emphasis, int lane) --{ -- switch (lane) { -- case 0: -- analogix_dp_set_lane0_pre_emphasis(dp, pre_emphasis); -- break; -- case 1: -- analogix_dp_set_lane1_pre_emphasis(dp, pre_emphasis); -- break; -- -- case 2: -- analogix_dp_set_lane2_pre_emphasis(dp, pre_emphasis); -- break; -- -- case 3: -- analogix_dp_set_lane3_pre_emphasis(dp, pre_emphasis); -- break; -- } --} -- - static int analogix_dp_link_start(struct analogix_dp_device *dp) - { - u8 buf[4]; -- int lane, lane_count, pll_tries, retval; -+ int lane, lane_count, retval; - - lane_count = dp->link_train.lane_count; - -@@ -278,6 +330,14 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp) - retval = drm_dp_dpcd_write(&dp->aux, DP_LINK_BW_SET, buf, 2); - if (retval < 0) - return retval; -+ -+ /* Spread AMP if required, enable 8b/10b coding */ -+ buf[0] = analogix_dp_ssc_supported(dp) ? DP_SPREAD_AMP_0_5 : 0; -+ buf[1] = DP_SET_ANSI_8B10B; -+ retval = drm_dp_dpcd_write(&dp->aux, DP_DOWNSPREAD_CTRL, buf, 2); -+ if (retval < 0) -+ return retval; -+ - /* set enhanced mode if available */ - retval = analogix_dp_set_enhanced_mode(dp); - if (retval < 0) { -@@ -285,22 +345,12 @@ static int analogix_dp_link_start(struct analogix_dp_device *dp) - return retval; - } - -- /* Set TX pre-emphasis to minimum */ -+ /* Set TX voltage-swing and pre-emphasis to minimum */ - for (lane = 0; lane < lane_count; lane++) -- analogix_dp_set_lane_lane_pre_emphasis(dp, -- PRE_EMPHASIS_LEVEL_0, lane); -- -- /* Wait for PLL lock */ -- pll_tries = 0; -- while (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { -- if (pll_tries == DP_TIMEOUT_LOOP_COUNT) { -- dev_err(dp->dev, "Wait for PLL lock timed out\n"); -- return -ETIMEDOUT; -- } -- -- pll_tries++; -- usleep_range(90, 120); -- } -+ dp->link_train.training_lane[lane] = -+ DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | -+ DP_TRAIN_PRE_EMPH_LEVEL_0; -+ analogix_dp_set_lane_link_training(dp); - - /* Set training pattern 1 */ - analogix_dp_set_training_pattern(dp, TRAINING_PTN1); -@@ -383,54 +433,6 @@ static unsigned char analogix_dp_get_adjust_request_pre_emphasis( - return ((link_value >> shift) & 0xc) >> 2; - } - --static void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp, -- u8 training_lane_set, int lane) --{ -- switch (lane) { -- case 0: -- analogix_dp_set_lane0_link_training(dp, training_lane_set); -- break; -- case 1: -- analogix_dp_set_lane1_link_training(dp, training_lane_set); -- break; -- -- case 2: -- analogix_dp_set_lane2_link_training(dp, training_lane_set); -- break; -- -- case 3: -- analogix_dp_set_lane3_link_training(dp, training_lane_set); -- break; -- } --} -- --static unsigned int --analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, -- int lane) --{ -- u32 reg; -- -- switch (lane) { -- case 0: -- reg = analogix_dp_get_lane0_link_training(dp); -- break; -- case 1: -- reg = analogix_dp_get_lane1_link_training(dp); -- break; -- case 2: -- reg = analogix_dp_get_lane2_link_training(dp); -- break; -- case 3: -- reg = analogix_dp_get_lane3_link_training(dp); -- break; -- default: -- WARN_ON(1); -- return 0; -- } -- -- return reg; --} -- - static void analogix_dp_reduce_link_rate(struct analogix_dp_device *dp) - { - analogix_dp_training_pattern_dis(dp); -@@ -463,11 +465,25 @@ static void analogix_dp_get_adjust_training_lane(struct analogix_dp_device *dp, - } - } - -+static bool analogix_dp_tps3_supported(struct analogix_dp_device *dp) -+{ -+ bool source_tps3_supported, sink_tps3_supported; -+ u8 dpcd = 0; -+ -+ source_tps3_supported = -+ dp->video_info.max_link_rate == DP_LINK_BW_5_4; -+ drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &dpcd); -+ sink_tps3_supported = dpcd & DP_TPS3_SUPPORTED; -+ -+ return source_tps3_supported && sink_tps3_supported; -+} -+ - static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) - { - int lane, lane_count, retval; - u8 voltage_swing, pre_emphasis, training_lane; - u8 link_status[2], adjust_request[2]; -+ u8 training_pattern = TRAINING_PTN2; - - usleep_range(100, 101); - -@@ -483,12 +499,16 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) - return retval; - - if (analogix_dp_clock_recovery_ok(link_status, lane_count) == 0) { -- /* set training pattern 2 for EQ */ -- analogix_dp_set_training_pattern(dp, TRAINING_PTN2); -+ if (analogix_dp_tps3_supported(dp)) -+ training_pattern = TRAINING_PTN3; -+ -+ /* set training pattern for EQ */ -+ analogix_dp_set_training_pattern(dp, training_pattern); - - retval = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET, - DP_LINK_SCRAMBLING_DISABLE | -- DP_TRAINING_PATTERN_2); -+ (training_pattern == TRAINING_PTN3 ? -+ DP_TRAINING_PATTERN_3 : DP_TRAINING_PATTERN_2)); - if (retval < 0) - return retval; - -@@ -519,25 +539,23 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp) - return -EIO; - } - } -- } - -- analogix_dp_get_adjust_training_lane(dp, adjust_request); -- -- for (lane = 0; lane < lane_count; lane++) -- analogix_dp_set_lane_link_training(dp, -- dp->link_train.training_lane[lane], lane); -+ analogix_dp_get_adjust_training_lane(dp, adjust_request); -+ analogix_dp_set_lane_link_training(dp); - -- retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, -- dp->link_train.training_lane, lane_count); -- if (retval < 0) -- return retval; -+ retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, -+ dp->link_train.training_lane, -+ lane_count); -+ if (retval < 0) -+ return retval; -+ } - - return 0; - } - - static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) - { -- int lane, lane_count, retval; -+ int lane_count, retval; - u32 reg; - u8 link_align, link_status[2], adjust_request[2]; - -@@ -597,9 +615,7 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) - return -EIO; - } - -- for (lane = 0; lane < lane_count; lane++) -- analogix_dp_set_lane_link_training(dp, -- dp->link_train.training_lane[lane], lane); -+ analogix_dp_set_lane_link_training(dp); - - retval = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET, - dp->link_train.training_lane, lane_count); -@@ -609,10 +625,11 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp) - return 0; - } - --static void analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp, -- u8 *bandwidth) -+static int analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp, -+ u8 *bandwidth) - { - u8 data; -+ int ret; - - /* - * For DP rev.1.1, Maximum link rate of Main Link lanes -@@ -620,28 +637,41 @@ static void analogix_dp_get_max_rx_bandwidth(struct analogix_dp_device *dp, - * For DP rev.1.2, Maximum link rate of Main Link lanes - * 0x06 = 1.62 Gbps, 0x0a = 2.7 Gbps, 0x14 = 5.4Gbps - */ -- drm_dp_dpcd_readb(&dp->aux, DP_MAX_LINK_RATE, &data); -+ ret = drm_dp_dpcd_readb(&dp->aux, DP_MAX_LINK_RATE, &data); -+ if (ret < 0) -+ return ret; -+ - *bandwidth = data; -+ -+ return 0; - } - --static void analogix_dp_get_max_rx_lane_count(struct analogix_dp_device *dp, -- u8 *lane_count) -+static int analogix_dp_get_max_rx_lane_count(struct analogix_dp_device *dp, -+ u8 *lane_count) - { - u8 data; -+ int ret; - - /* - * For DP rev.1.1, Maximum number of Main Link lanes - * 0x01 = 1 lane, 0x02 = 2 lanes, 0x04 = 4 lanes - */ -- drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data); -+ ret = drm_dp_dpcd_readb(&dp->aux, DP_MAX_LANE_COUNT, &data); -+ if (ret < 0) -+ return ret; -+ - *lane_count = DPCD_MAX_LANE_COUNT(data); -+ -+ return 0; - } - - static int analogix_dp_full_link_train(struct analogix_dp_device *dp, - u32 max_lanes, u32 max_rate) - { -+ struct video_info *video = &dp->video_info; - int retval = 0; - bool training_finished = false; -+ u8 dpcd; - - /* - * MACRO_RST must be applied after the PLL_LOCK to avoid -@@ -667,6 +697,16 @@ static int analogix_dp_full_link_train(struct analogix_dp_device *dp, - dp->link_train.lane_count = (u8)LANE_COUNT1; - } - -+ if (!analogix_dp_bandwidth_ok(dp, &video->mode, -+ drm_dp_bw_code_to_link_rate(dp->link_train.link_rate), -+ dp->link_train.lane_count)) { -+ dev_err(dp->dev, "bandwidth overflow\n"); -+ return -EINVAL; -+ } -+ -+ drm_dp_dpcd_readb(&dp->aux, DP_MAX_DOWNSPREAD, &dpcd); -+ dp->link_train.ssc = !!(dpcd & DP_MAX_DOWNSPREAD_0_5); -+ - /* Setup TX lane count & rate */ - if (dp->link_train.lane_count > max_lanes) - dp->link_train.lane_count = max_lanes; -@@ -711,27 +751,15 @@ static int analogix_dp_full_link_train(struct analogix_dp_device *dp, - - static int analogix_dp_fast_link_train(struct analogix_dp_device *dp) - { -- int i, ret; -+ int ret; - u8 link_align, link_status[2]; -- enum pll_status status; - - analogix_dp_reset_macro(dp); - - analogix_dp_set_link_bandwidth(dp, dp->link_train.link_rate); - analogix_dp_set_lane_count(dp, dp->link_train.lane_count); -- -- for (i = 0; i < dp->link_train.lane_count; i++) { -- analogix_dp_set_lane_link_training(dp, -- dp->link_train.training_lane[i], i); -- } -- -- ret = readx_poll_timeout(analogix_dp_get_pll_lock_status, dp, status, -- status != PLL_UNLOCKED, 120, -- 120 * DP_TIMEOUT_LOOP_COUNT); -- if (ret) { -- DRM_DEV_ERROR(dp->dev, "Wait for pll lock failed %d\n", ret); -- return ret; -- } -+ analogix_dp_set_lane_link_training(dp); -+ analogix_dp_enable_enhanced_mode(dp, dp->link_train.enhanced_framing); - - /* source Set training pattern 1 */ - analogix_dp_set_training_pattern(dp, TRAINING_PTN1); -@@ -742,7 +770,6 @@ static int analogix_dp_fast_link_train(struct analogix_dp_device *dp) - /* From DP spec, pattern must be on-screen for a minimum 500us */ - usleep_range(500, 600); - -- /* TODO: enhanced_mode?*/ - analogix_dp_set_training_pattern(dp, DP_NONE); - - /* -@@ -884,25 +911,44 @@ static int analogix_dp_enable_scramble(struct analogix_dp_device *dp, - return ret < 0 ? ret : 0; - } - -+static irqreturn_t analogix_dp_hpd_irq_handler(int irq, void *arg) -+{ -+ struct analogix_dp_device *dp = arg; -+ -+ if (dp->drm_dev) -+ drm_helper_hpd_irq_event(dp->drm_dev); -+ -+ return IRQ_HANDLED; -+} -+ - static irqreturn_t analogix_dp_hardirq(int irq, void *arg) - { - struct analogix_dp_device *dp = arg; -- irqreturn_t ret = IRQ_NONE; - enum dp_irq_type irq_type; -+ int ret; -+ -+ ret = pm_runtime_get_sync(dp->dev); -+ if (ret < 0) -+ return IRQ_NONE; - - irq_type = analogix_dp_get_irq_type(dp); -- if (irq_type != DP_IRQ_TYPE_UNKNOWN) { -+ if (irq_type != DP_IRQ_TYPE_UNKNOWN) - analogix_dp_mute_hpd_interrupt(dp); -- ret = IRQ_WAKE_THREAD; -- } - -- return ret; -+ pm_runtime_put_sync(dp->dev); -+ -+ return IRQ_WAKE_THREAD; - } - - static irqreturn_t analogix_dp_irq_thread(int irq, void *arg) - { - struct analogix_dp_device *dp = arg; - enum dp_irq_type irq_type; -+ int ret; -+ -+ ret = pm_runtime_get_sync(dp->dev); -+ if (ret < 0) -+ return IRQ_NONE; - - irq_type = analogix_dp_get_irq_type(dp); - if (irq_type & DP_IRQ_TYPE_HP_CABLE_IN || -@@ -917,6 +963,8 @@ static irqreturn_t analogix_dp_irq_thread(int irq, void *arg) - analogix_dp_unmute_hpd_interrupt(dp); - } - -+ pm_runtime_put_sync(dp->dev); -+ - return IRQ_HANDLED; - } - -@@ -938,13 +986,12 @@ static int analogix_dp_fast_link_train_detection(struct analogix_dp_device *dp) - - static int analogix_dp_commit(struct analogix_dp_device *dp) - { -+ struct video_info *video = &dp->video_info; - int ret; - -- /* Keep the panel disabled while we configure video */ -- if (dp->plat_data->panel) { -- if (drm_panel_disable(dp->plat_data->panel)) -- DRM_ERROR("failed to disable the panel\n"); -- } -+ if (device_property_read_bool(dp->dev, "panel-self-test")) -+ return drm_dp_dpcd_writeb(&dp->aux, DP_EDP_CONFIGURATION_SET, -+ DP_PANEL_SELF_TEST_ENABLE); - - ret = analogix_dp_train_link(dp); - if (ret) { -@@ -959,21 +1006,17 @@ static int analogix_dp_commit(struct analogix_dp_device *dp) - } - - analogix_dp_init_video(dp); -+ analogix_dp_set_video_format(dp); -+ -+ if (video->video_bist_enable) -+ analogix_dp_video_bist_enable(dp); -+ - ret = analogix_dp_config_video(dp); - if (ret) { - dev_err(dp->dev, "unable to config video\n"); - return ret; - } - -- /* Safe to enable the panel now */ -- if (dp->plat_data->panel) { -- ret = drm_panel_enable(dp->plat_data->panel); -- if (ret) { -- DRM_ERROR("failed to enable the panel\n"); -- return ret; -- } -- } -- - /* Check whether panel supports fast training */ - ret = analogix_dp_fast_link_train_detection(dp); - if (ret) -@@ -1058,66 +1101,18 @@ static int analogix_dp_disable_psr(struct analogix_dp_device *dp) - return analogix_dp_send_psr_spd(dp, &psr_vsc, true); - } - --/* -- * This function is a bit of a catch-all for panel preparation, hopefully -- * simplifying the logic of functions that need to prepare/unprepare the panel -- * below. -- * -- * If @prepare is true, this function will prepare the panel. Conversely, if it -- * is false, the panel will be unprepared. -- * -- * If @is_modeset_prepare is true, the function will disregard the current state -- * of the panel and either prepare/unprepare the panel based on @prepare. Once -- * it finishes, it will update dp->panel_is_modeset to reflect the current state -- * of the panel. -- */ --static int analogix_dp_prepare_panel(struct analogix_dp_device *dp, -- bool prepare, bool is_modeset_prepare) --{ -- int ret = 0; -- -- if (!dp->plat_data->panel) -- return 0; -- -- mutex_lock(&dp->panel_lock); -- -- /* -- * Exit early if this is a temporary prepare/unprepare and we're already -- * modeset (since we neither want to prepare twice or unprepare early). -- */ -- if (dp->panel_is_modeset && !is_modeset_prepare) -- goto out; -- -- if (prepare) -- ret = drm_panel_prepare(dp->plat_data->panel); -- else -- ret = drm_panel_unprepare(dp->plat_data->panel); -- -- if (ret) -- goto out; -- -- if (is_modeset_prepare) -- dp->panel_is_modeset = prepare; -- --out: -- mutex_unlock(&dp->panel_lock); -- return ret; --} -- - static int analogix_dp_get_modes(struct drm_connector *connector) - { - struct analogix_dp_device *dp = to_dp(connector); - struct edid *edid; -- int ret, num_modes = 0; -+ int num_modes = 0; - -- if (dp->plat_data->panel) { -+ if (dp->plat_data->panel) - num_modes += drm_panel_get_modes(dp->plat_data->panel, connector); -- } else { -- ret = analogix_dp_prepare_panel(dp, true, false); -- if (ret) { -- DRM_ERROR("Failed to prepare panel (%d)\n", ret); -- return 0; -- } -+ -+ if (!num_modes) { -+ if (dp->plat_data->panel) -+ analogix_dp_panel_prepare(dp); - - pm_runtime_get_sync(dp->dev); - edid = drm_get_edid(connector, &dp->aux.ddc); -@@ -1128,15 +1123,18 @@ static int analogix_dp_get_modes(struct drm_connector *connector) - num_modes += drm_add_edid_modes(&dp->connector, edid); - kfree(edid); - } -- -- ret = analogix_dp_prepare_panel(dp, false, false); -- if (ret) -- DRM_ERROR("Failed to unprepare panel (%d)\n", ret); - } - - if (dp->plat_data->get_modes) - num_modes += dp->plat_data->get_modes(dp->plat_data, connector); - -+ if (num_modes > 0 && dp->plat_data->split_mode) { -+ struct drm_display_mode *mode; -+ -+ list_for_each_entry(mode, &connector->probed_modes, head) -+ dp->plat_data->convert_to_split_mode(mode); -+ } -+ - return num_modes; - } - -@@ -1182,34 +1180,52 @@ static const struct drm_connector_helper_funcs analogix_dp_connector_helper_func - }; - - static enum drm_connector_status --analogix_dp_detect(struct drm_connector *connector, bool force) -+analogix_dp_detect(struct analogix_dp_device *dp) - { -- struct analogix_dp_device *dp = to_dp(connector); - enum drm_connector_status status = connector_status_disconnected; - int ret; - - if (dp->plat_data->panel) -- return connector_status_connected; -+ analogix_dp_panel_prepare(dp); - -- ret = analogix_dp_prepare_panel(dp, true, false); -- if (ret) { -- DRM_ERROR("Failed to prepare panel (%d)\n", ret); -- return connector_status_disconnected; -- } -+ pm_runtime_get_sync(dp->dev); -+ -+ if (!analogix_dp_detect_hpd(dp)) { -+ ret = analogix_dp_get_max_rx_bandwidth(dp, &dp->link_train.link_rate); -+ if (ret) { -+ dev_err(dp->dev, "failed to read max link rate\n"); -+ goto out; -+ } -+ -+ ret = analogix_dp_get_max_rx_lane_count(dp, &dp->link_train.lane_count); -+ if (ret) { -+ dev_err(dp->dev, "failed to read max lane count\n"); -+ goto out; -+ } - -- if (!analogix_dp_detect_hpd(dp)) - status = connector_status_connected; -+ } - -- ret = analogix_dp_prepare_panel(dp, false, false); -- if (ret) -- DRM_ERROR("Failed to unprepare panel (%d)\n", ret); -+out: -+ pm_runtime_put(dp->dev); - - return status; - } - -+static enum drm_connector_status -+analogix_dp_connector_detect(struct drm_connector *connector, bool force) -+{ -+ struct analogix_dp_device *dp = to_dp(connector); -+ -+ if (dp->plat_data->right && analogix_dp_detect(dp->plat_data->right) != connector_status_connected) -+ return connector_status_disconnected; -+ -+ return analogix_dp_detect(dp); -+} -+ - static const struct drm_connector_funcs analogix_dp_connector_funcs = { - .fill_modes = drm_helper_probe_single_connector_modes, -- .detect = analogix_dp_detect, -+ .detect = analogix_dp_connector_detect, - .destroy = drm_connector_cleanup, - .reset = drm_atomic_helper_connector_reset, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, -@@ -1224,10 +1240,8 @@ static int analogix_dp_bridge_attach(struct drm_bridge *bridge, - struct drm_connector *connector = NULL; - int ret = 0; - -- if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) { -- DRM_ERROR("Fix bridge driver to make connector optional!"); -- return -EINVAL; -- } -+ if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) -+ return 0; - - if (!bridge->encoder) { - DRM_ERROR("Parent encoder object not found"); -@@ -1268,6 +1282,14 @@ static int analogix_dp_bridge_attach(struct drm_bridge *bridge, - return 0; - } - -+static void analogix_dp_bridge_detach(struct drm_bridge *bridge) -+{ -+ struct analogix_dp_device *dp = bridge->driver_private; -+ -+ if (dp->plat_data->detach) -+ dp->plat_data->detach(dp->plat_data, bridge); -+} -+ - static - struct drm_crtc *analogix_dp_get_new_crtc(struct analogix_dp_device *dp, - struct drm_atomic_state *state) -@@ -1295,20 +1317,20 @@ analogix_dp_bridge_atomic_pre_enable(struct drm_bridge *bridge, - struct analogix_dp_device *dp = bridge->driver_private; - struct drm_crtc *crtc; - struct drm_crtc_state *old_crtc_state; -- int ret; - -- crtc = analogix_dp_get_new_crtc(dp, old_state); -- if (!crtc) -- return; -+ if (dp->psr_supported) { -+ crtc = analogix_dp_get_new_crtc(dp, old_state); -+ if (!crtc) -+ return; - -- old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); -- /* Don't touch the panel if we're coming back from PSR */ -- if (old_crtc_state && old_crtc_state->self_refresh_active) -- return; -+ old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); -+ /* Don't touch the panel if we're coming back from PSR */ -+ if (old_crtc_state && old_crtc_state->self_refresh_active) -+ return; -+ } - -- ret = analogix_dp_prepare_panel(dp, true, true); -- if (ret) -- DRM_ERROR("failed to setup the panel ret = %d\n", ret); -+ if (dp->plat_data->panel) -+ analogix_dp_panel_prepare(dp); - } - - static int analogix_dp_set_bridge(struct analogix_dp_device *dp) -@@ -1317,16 +1339,10 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) - - pm_runtime_get_sync(dp->dev); - -- ret = clk_prepare_enable(dp->clock); -- if (ret < 0) { -- DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret); -- goto out_dp_clk_pre; -- } -- - if (dp->plat_data->power_on_start) - dp->plat_data->power_on_start(dp->plat_data); - -- phy_power_on(dp->phy); -+ analogix_dp_phy_power_on(dp); - - ret = analogix_dp_init_dp(dp); - if (ret) -@@ -1344,11 +1360,14 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) - } - - ret = analogix_dp_commit(dp); -- if (ret) { -+ if (ret < 0) { - DRM_ERROR("dp commit error, ret = %d\n", ret); - goto out_dp_init; - } - -+ if (dp->plat_data->panel) -+ drm_panel_enable(dp->plat_data->panel); -+ - if (dp->plat_data->power_on_end) - dp->plat_data->power_on_end(dp->plat_data); - -@@ -1356,11 +1375,9 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp) - return 0; - - out_dp_init: -- phy_power_off(dp->phy); -+ analogix_dp_phy_power_off(dp); - if (dp->plat_data->power_off) - dp->plat_data->power_off(dp->plat_data); -- clk_disable_unprepare(dp->clock); --out_dp_clk_pre: - pm_runtime_put_sync(dp->dev); - - return ret; -@@ -1377,17 +1394,19 @@ analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge, - int timeout_loop = 0; - int ret; - -- crtc = analogix_dp_get_new_crtc(dp, old_state); -- if (!crtc) -- return; -+ if (dp->psr_supported) { -+ crtc = analogix_dp_get_new_crtc(dp, old_state); -+ if (!crtc) -+ return; - -- old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); -- /* Not a full enable, just disable PSR and continue */ -- if (old_crtc_state && old_crtc_state->self_refresh_active) { -- ret = analogix_dp_disable_psr(dp); -- if (ret) -- DRM_ERROR("Failed to disable psr %d\n", ret); -- return; -+ old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc); -+ /* Not a full enable, just disable PSR and continue */ -+ if (old_crtc_state && old_crtc_state->self_refresh_active) { -+ ret = analogix_dp_disable_psr(dp); -+ if (ret) -+ DRM_ERROR("Failed to disable psr %d\n", ret); -+ return; -+ } - } - - if (dp->dpms_mode == DRM_MODE_DPMS_ON) -@@ -1409,7 +1428,6 @@ analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge, - static void analogix_dp_bridge_disable(struct drm_bridge *bridge) - { - struct analogix_dp_device *dp = bridge->driver_private; -- int ret; - - if (dp->dpms_mode != DRM_MODE_DPMS_ON) - return; -@@ -1426,16 +1444,14 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge) - if (dp->plat_data->power_off) - dp->plat_data->power_off(dp->plat_data); - -+ analogix_dp_reset_aux(dp); - analogix_dp_set_analog_power_down(dp, POWER_ALL, 1); -- phy_power_off(dp->phy); -- -- clk_disable_unprepare(dp->clock); -+ analogix_dp_phy_power_off(dp); - - pm_runtime_put_sync(dp->dev); - -- ret = analogix_dp_prepare_panel(dp, false, true); -- if (ret) -- DRM_ERROR("failed to setup the panel ret = %d\n", ret); -+ if (dp->plat_data->panel) -+ analogix_dp_panel_unprepare(dp); - - dp->fast_train_enable = false; - dp->psr_supported = false; -@@ -1492,14 +1508,19 @@ analogix_dp_bridge_atomic_post_disable(struct drm_bridge *bridge, - - static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge, - const struct drm_display_mode *orig_mode, -- const struct drm_display_mode *mode) -+ const struct drm_display_mode *adj_mode) - { - struct analogix_dp_device *dp = bridge->driver_private; - struct drm_display_info *display_info = &dp->connector.display_info; - struct video_info *video = &dp->video_info; -+ struct drm_display_mode *mode = &video->mode; - struct device_node *dp_node = dp->dev->of_node; - int vic; - -+ drm_mode_copy(mode, adj_mode); -+ if (dp->plat_data->split_mode) -+ dp->plat_data->convert_to_origin_mode(mode); -+ - /* Input video interlaces & hsync pol & vsync pol */ - video->interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE); - video->v_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NVSYNC); -@@ -1567,6 +1588,27 @@ static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge, - video->interlaced = true; - } - -+static enum drm_mode_status -+analogix_dp_bridge_mode_valid(struct drm_bridge *bridge, -+ const struct drm_display_info *info, -+ const struct drm_display_mode *mode) -+{ -+ struct analogix_dp_device *dp = bridge->driver_private; -+ struct drm_display_mode m; -+ -+ drm_mode_copy(&m, mode); -+ -+ if (dp->plat_data->split_mode) -+ dp->plat_data->convert_to_origin_mode(&m); -+ -+ if (!analogix_dp_bandwidth_ok(dp, &m, -+ drm_dp_bw_code_to_link_rate(dp->link_train.link_rate), -+ dp->link_train.lane_count)) -+ return MODE_BAD; -+ -+ return MODE_OK; -+} -+ - static const struct drm_bridge_funcs analogix_dp_bridge_funcs = { - .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, -@@ -1577,29 +1619,30 @@ static const struct drm_bridge_funcs analogix_dp_bridge_funcs = { - .atomic_post_disable = analogix_dp_bridge_atomic_post_disable, - .mode_set = analogix_dp_bridge_mode_set, - .attach = analogix_dp_bridge_attach, -+ .detach = analogix_dp_bridge_detach, -+ .mode_valid = analogix_dp_bridge_mode_valid, - }; - --static int analogix_dp_create_bridge(struct drm_device *drm_dev, -- struct analogix_dp_device *dp) -+static int analogix_dp_bridge_init(struct analogix_dp_device *dp) - { -- struct drm_bridge *bridge; -+ struct drm_bridge *bridge = &dp->bridge; - int ret; - -- bridge = devm_kzalloc(drm_dev->dev, sizeof(*bridge), GFP_KERNEL); -- if (!bridge) { -- DRM_ERROR("failed to allocate for drm bridge\n"); -- return -ENOMEM; -+ if (!dp->plat_data->left) { -+ ret = drm_bridge_attach(dp->encoder, bridge, NULL, 0); -+ if (ret) { -+ DRM_ERROR("failed to attach drm bridge\n"); -+ return ret; -+ } - } - -- dp->bridge = bridge; -+ if (dp->plat_data->right) { -+ struct analogix_dp_device *secondary = dp->plat_data->right; - -- bridge->driver_private = dp; -- bridge->funcs = &analogix_dp_bridge_funcs; -- -- ret = drm_bridge_attach(dp->encoder, bridge, NULL, 0); -- if (ret) { -- DRM_ERROR("failed to attach drm bridge\n"); -- return -EINVAL; -+ ret = drm_bridge_attach(dp->encoder, &secondary->bridge, bridge, -+ DRM_BRIDGE_ATTACH_NO_CONNECTOR); -+ if (ret) -+ return ret; - } - - return 0; -@@ -1612,7 +1655,7 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) - - switch (dp->plat_data->dev_type) { - case RK3288_DP: -- case RK3399_EDP: -+ case RK3568_EDP: - /* - * Like Rk3288 DisplayPort TRM indicate that "Main link - * containing 4 physical lanes of 2.7/1.62 Gbps/lane". -@@ -1620,6 +1663,11 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) - video_info->max_link_rate = 0x0A; - video_info->max_lane_count = 0x04; - break; -+ case RK3399_EDP: -+ case RK3588_EDP: -+ video_info->max_link_rate = 0x14; -+ video_info->max_lane_count = 0x04; -+ break; - case EXYNOS_DP: - /* - * NOTE: those property parseing code is used for -@@ -1632,6 +1680,9 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp) - break; - } - -+ video_info->video_bist_enable = -+ of_property_read_bool(dp_node, "analogix,video-bist-enable"); -+ - return 0; - } - -@@ -1643,13 +1694,72 @@ static ssize_t analogix_dpaux_transfer(struct drm_dp_aux *aux, - return analogix_dp_transfer(dp, msg); - } - -+int analogix_dp_audio_hw_params(struct analogix_dp_device *dp, -+ struct hdmi_codec_daifmt *daifmt, -+ struct hdmi_codec_params *params) -+{ -+ switch (daifmt->fmt) { -+ case HDMI_SPDIF: -+ analogix_dp_audio_config_spdif(dp); -+ break; -+ case HDMI_I2S: -+ analogix_dp_audio_config_i2s(dp); -+ break; -+ default: -+ DRM_DEV_ERROR(dp->dev, "invalid daifmt %d\n", daifmt->fmt); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(analogix_dp_audio_hw_params); -+ -+void analogix_dp_audio_shutdown(struct analogix_dp_device *dp) -+{ -+ analogix_dp_audio_disable(dp); -+} -+EXPORT_SYMBOL_GPL(analogix_dp_audio_shutdown); -+ -+int analogix_dp_audio_startup(struct analogix_dp_device *dp) -+{ -+ analogix_dp_audio_enable(dp); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(analogix_dp_audio_startup); -+ -+int analogix_dp_audio_get_eld(struct analogix_dp_device *dp, u8 *buf, size_t len) -+{ -+ memcpy(buf, dp->connector.eld, min(sizeof(dp->connector.eld), len)); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(analogix_dp_audio_get_eld); -+ -+int analogix_dp_loader_protect(struct analogix_dp_device *dp) -+{ -+ int ret; -+ -+ ret = pm_runtime_resume_and_get(dp->dev); -+ if (ret) { -+ dev_err(dp->dev, "failed to get runtime PM: %d\n", ret); -+ return ret; -+ } -+ -+ analogix_dp_phy_power_on(dp); -+ -+ dp->dpms_mode = DRM_MODE_DPMS_ON; -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(analogix_dp_loader_protect); -+ - struct analogix_dp_device * - analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) - { - struct platform_device *pdev = to_platform_device(dev); - struct analogix_dp_device *dp; - struct resource *res; -- unsigned int irq_flags; - int ret; - - if (!plat_data) { -@@ -1665,7 +1775,7 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) - dp->dpms_mode = DRM_MODE_DPMS_OFF; - - mutex_init(&dp->panel_lock); -- dp->panel_is_modeset = false; -+ dp->panel_is_prepared = false; - - /* - * platform dp driver need containor_of the plat_data to get -@@ -1694,13 +1804,13 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) - } - } - -- dp->clock = devm_clk_get(&pdev->dev, "dp"); -- if (IS_ERR(dp->clock)) { -- dev_err(&pdev->dev, "failed to get clock\n"); -- return ERR_CAST(dp->clock); -+ ret = devm_clk_bulk_get_all(dev, &dp->clks); -+ if (ret < 0) { -+ dev_err(dev, "failed to get clocks %d\n", ret); -+ return ERR_PTR(ret); - } - -- clk_prepare_enable(dp->clock); -+ dp->nr_clks = ret; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - -@@ -1722,34 +1832,38 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data) - } - - if (dp->hpd_gpiod) { -- /* -- * Set up the hotplug GPIO from the device tree as an interrupt. -- * Simply specifying a different interrupt in the device tree -- * doesn't work since we handle hotplug rather differently when -- * using a GPIO. We also need the actual GPIO specifier so -- * that we can get the current state of the GPIO. -- */ -- dp->irq = gpiod_to_irq(dp->hpd_gpiod); -- irq_flags = IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING; -- } else { -- dp->irq = platform_get_irq(pdev, 0); -- irq_flags = 0; -+ ret = devm_request_threaded_irq(dev, -+ gpiod_to_irq(dp->hpd_gpiod), -+ NULL, -+ analogix_dp_hpd_irq_handler, -+ IRQF_TRIGGER_RISING | -+ IRQF_TRIGGER_FALLING | -+ IRQF_ONESHOT, -+ "analogix-hpd", dp); -+ if (ret) { -+ dev_err(dev, "failed to request hpd IRQ: %d\n", ret); -+ return ERR_PTR(ret); -+ } - } - -+ dp->irq = platform_get_irq(pdev, 0); - if (dp->irq == -ENXIO) { - dev_err(&pdev->dev, "failed to get irq\n"); - return ERR_PTR(-ENODEV); - } - -+ irq_set_status_flags(dp->irq, IRQ_NOAUTOEN); - ret = devm_request_threaded_irq(&pdev->dev, dp->irq, - analogix_dp_hardirq, - analogix_dp_irq_thread, -- irq_flags, "analogix-dp", dp); -+ 0, "analogix-dp", dp); - if (ret) { - dev_err(&pdev->dev, "failed to request irq\n"); - return ERR_PTR(ret); - } -- disable_irq(dp->irq); -+ -+ dp->bridge.driver_private = dp; -+ dp->bridge.funcs = &analogix_dp_bridge_funcs; - - return dp; - } -@@ -1772,9 +1886,9 @@ int analogix_dp_bind(struct analogix_dp_device *dp, struct drm_device *drm_dev) - - pm_runtime_enable(dp->dev); - -- ret = analogix_dp_create_bridge(drm_dev, dp); -+ ret = analogix_dp_bridge_init(dp); - if (ret) { -- DRM_ERROR("failed to create bridge (%d)\n", ret); -+ DRM_ERROR("failed to init bridge (%d)\n", ret); - goto err_disable_pm_runtime; - } - -@@ -1789,14 +1903,7 @@ EXPORT_SYMBOL_GPL(analogix_dp_bind); - - void analogix_dp_unbind(struct analogix_dp_device *dp) - { -- analogix_dp_bridge_disable(dp->bridge); - dp->connector.funcs->destroy(&dp->connector); -- -- if (dp->plat_data->panel) { -- if (drm_panel_unprepare(dp->plat_data->panel)) -- DRM_ERROR("failed to turnoff the panel\n"); -- } -- - drm_dp_aux_unregister(&dp->aux); - pm_runtime_disable(dp->dev); - } -@@ -1804,45 +1911,22 @@ EXPORT_SYMBOL_GPL(analogix_dp_unbind); - - void analogix_dp_remove(struct analogix_dp_device *dp) - { -- clk_disable_unprepare(dp->clock); - } - EXPORT_SYMBOL_GPL(analogix_dp_remove); - --#ifdef CONFIG_PM --int analogix_dp_suspend(struct analogix_dp_device *dp) -+int analogix_dp_runtime_suspend(struct analogix_dp_device *dp) - { -- clk_disable_unprepare(dp->clock); -- -- if (dp->plat_data->panel) { -- if (drm_panel_unprepare(dp->plat_data->panel)) -- DRM_ERROR("failed to turnoff the panel\n"); -- } -+ clk_bulk_disable_unprepare(dp->nr_clks, dp->clks); - - return 0; - } --EXPORT_SYMBOL_GPL(analogix_dp_suspend); -+EXPORT_SYMBOL_GPL(analogix_dp_runtime_suspend); - --int analogix_dp_resume(struct analogix_dp_device *dp) -+int analogix_dp_runtime_resume(struct analogix_dp_device *dp) - { -- int ret; -- -- ret = clk_prepare_enable(dp->clock); -- if (ret < 0) { -- DRM_ERROR("Failed to prepare_enable the clock clk [%d]\n", ret); -- return ret; -- } -- -- if (dp->plat_data->panel) { -- if (drm_panel_prepare(dp->plat_data->panel)) { -- DRM_ERROR("failed to setup the panel\n"); -- return -EBUSY; -- } -- } -- -- return 0; -+ return clk_bulk_prepare_enable(dp->nr_clks, dp->clks); - } --EXPORT_SYMBOL_GPL(analogix_dp_resume); --#endif -+EXPORT_SYMBOL_GPL(analogix_dp_runtime_resume); - - int analogix_dp_start_crc(struct drm_connector *connector) - { -diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h -index c051502d7..28f2aaa14 100644 ---- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h -+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h -@@ -10,6 +10,7 @@ - #define _ANALOGIX_DP_CORE_H - - #include -+#include - #include - - #define DP_TIMEOUT_LOOP_COUNT 100 -@@ -69,6 +70,7 @@ enum pattern_set { - D10_2, - TRAINING_PTN1, - TRAINING_PTN2, -+ TRAINING_PTN3, - DP_NONE - }; - -@@ -129,6 +131,7 @@ enum dp_irq_type { - - struct video_info { - char *name; -+ struct drm_display_mode mode; - - bool h_sync_polarity; - bool v_sync_polarity; -@@ -141,6 +144,8 @@ struct video_info { - - int max_link_rate; - enum link_lane_count_type max_lane_count; -+ -+ bool video_bist_enable; - }; - - struct link_train { -@@ -150,6 +155,8 @@ struct link_train { - u8 link_rate; - u8 lane_count; - u8 training_lane[4]; -+ bool ssc; -+ bool enhanced_framing; - - enum link_training_state lt_state; - }; -@@ -159,15 +166,17 @@ struct analogix_dp_device { - struct device *dev; - struct drm_device *drm_dev; - struct drm_connector connector; -- struct drm_bridge *bridge; -+ struct drm_bridge bridge; - struct drm_dp_aux aux; -- struct clk *clock; -+ struct clk_bulk_data *clks; -+ int nr_clks; - unsigned int irq; - void __iomem *reg_base; - - struct video_info video_info; - struct link_train link_train; - struct phy *phy; -+ bool phy_enabled; - int dpms_mode; - struct gpio_desc *hpd_gpiod; - bool force_hpd; -@@ -175,7 +184,7 @@ struct analogix_dp_device { - bool psr_supported; - - struct mutex panel_lock; -- bool panel_is_modeset; -+ bool panel_is_prepared; - - struct analogix_dp_plat_data *plat_data; - }; -@@ -213,26 +222,8 @@ void analogix_dp_enable_enhanced_mode(struct analogix_dp_device *dp, - bool enable); - void analogix_dp_set_training_pattern(struct analogix_dp_device *dp, - enum pattern_set pattern); --void analogix_dp_set_lane0_pre_emphasis(struct analogix_dp_device *dp, -- u32 level); --void analogix_dp_set_lane1_pre_emphasis(struct analogix_dp_device *dp, -- u32 level); --void analogix_dp_set_lane2_pre_emphasis(struct analogix_dp_device *dp, -- u32 level); --void analogix_dp_set_lane3_pre_emphasis(struct analogix_dp_device *dp, -- u32 level); --void analogix_dp_set_lane0_link_training(struct analogix_dp_device *dp, -- u32 training_lane); --void analogix_dp_set_lane1_link_training(struct analogix_dp_device *dp, -- u32 training_lane); --void analogix_dp_set_lane2_link_training(struct analogix_dp_device *dp, -- u32 training_lane); --void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp, -- u32 training_lane); --u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp); --u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp); --u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp); --u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp); -+void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp); -+u32 analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, u8 lane); - void analogix_dp_reset_macro(struct analogix_dp_device *dp); - void analogix_dp_init_video(struct analogix_dp_device *dp); - -@@ -255,5 +246,14 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, - struct dp_sdp *vsc, bool blocking); - ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - struct drm_dp_aux_msg *msg); -+void analogix_dp_set_video_format(struct analogix_dp_device *dp); -+void analogix_dp_video_bist_enable(struct analogix_dp_device *dp); -+bool analogix_dp_ssc_supported(struct analogix_dp_device *dp); -+void analogix_dp_phy_power_on(struct analogix_dp_device *dp); -+void analogix_dp_phy_power_off(struct analogix_dp_device *dp); -+void analogix_dp_audio_config_spdif(struct analogix_dp_device *dp); -+void analogix_dp_audio_config_i2s(struct analogix_dp_device *dp); -+void analogix_dp_audio_enable(struct analogix_dp_device *dp); -+void analogix_dp_audio_disable(struct analogix_dp_device *dp); - - #endif /* _ANALOGIX_DP_CORE_H */ -diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c -index 914c569ab..e76c66c7c 100644 ---- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c -+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c -@@ -11,6 +11,7 @@ - #include - #include - #include -+#include - - #include - -@@ -21,20 +22,37 @@ - #define COMMON_INT_MASK_2 0 - #define COMMON_INT_MASK_3 0 - #define COMMON_INT_MASK_4 (HOTPLUG_CHG | HPD_LOST | PLUG) --#define INT_STA_MASK INT_HPD -+ -+static void analogix_dp_write(struct analogix_dp_device *dp, u32 reg, u32 val) -+{ -+ if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { -+ readl(dp->reg_base); -+ writel(val, dp->reg_base + reg); -+ } -+ -+ writel(val, dp->reg_base + reg); -+} -+ -+static u32 analogix_dp_read(struct analogix_dp_device *dp, u32 reg) -+{ -+ if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) -+ readl(dp->reg_base + reg); -+ -+ return readl(dp->reg_base + reg); -+} - - void analogix_dp_enable_video_mute(struct analogix_dp_device *dp, bool enable) - { - u32 reg; - - if (enable) { -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); - reg |= HDCP_VIDEO_MUTE; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); - } else { -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); - reg &= ~HDCP_VIDEO_MUTE; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); - } - } - -@@ -42,9 +60,9 @@ void analogix_dp_stop_video(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); - reg &= ~VIDEO_EN; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); - } - - void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable) -@@ -58,7 +76,7 @@ void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable) - reg = LANE3_MAP_LOGIC_LANE_3 | LANE2_MAP_LOGIC_LANE_2 | - LANE1_MAP_LOGIC_LANE_1 | LANE0_MAP_LOGIC_LANE_0; - -- writel(reg, dp->reg_base + ANALOGIX_DP_LANE_MAP); -+ analogix_dp_write(dp, ANALOGIX_DP_LANE_MAP, reg); - } - - void analogix_dp_init_analog_param(struct analogix_dp_device *dp) -@@ -66,53 +84,54 @@ void analogix_dp_init_analog_param(struct analogix_dp_device *dp) - u32 reg; - - reg = TX_TERMINAL_CTRL_50_OHM; -- writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_1, reg); - - reg = SEL_24M | TX_DVDD_BIT_1_0625V; -- writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_2); -+ analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_2, reg); - - if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { - reg = REF_CLK_24M; - if (dp->plat_data->dev_type == RK3288_DP) - reg ^= REF_CLK_MASK; - -- writel(reg, dp->reg_base + ANALOGIX_DP_PLL_REG_1); -- writel(0x95, dp->reg_base + ANALOGIX_DP_PLL_REG_2); -- writel(0x40, dp->reg_base + ANALOGIX_DP_PLL_REG_3); -- writel(0x58, dp->reg_base + ANALOGIX_DP_PLL_REG_4); -- writel(0x22, dp->reg_base + ANALOGIX_DP_PLL_REG_5); -+ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_1, reg); -+ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_2, 0x99); -+ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_3, 0x40); -+ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_4, 0x58); -+ analogix_dp_write(dp, ANALOGIX_DP_PLL_REG_5, 0x22); -+ analogix_dp_write(dp, ANALOGIX_DP_BIAS, 0x44); - } - - reg = DRIVE_DVDD_BIT_1_0625V | VCO_BIT_600_MICRO; -- writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_3); -+ analogix_dp_write(dp, ANALOGIX_DP_ANALOG_CTL_3, reg); - - reg = PD_RING_OSC | AUX_TERMINAL_CTRL_50_OHM | - TX_CUR1_2X | TX_CUR_16_MA; -- writel(reg, dp->reg_base + ANALOGIX_DP_PLL_FILTER_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_PLL_FILTER_CTL_1, reg); - - reg = CH3_AMP_400_MV | CH2_AMP_400_MV | - CH1_AMP_400_MV | CH0_AMP_400_MV; -- writel(reg, dp->reg_base + ANALOGIX_DP_TX_AMP_TUNING_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_TX_AMP_TUNING_CTL, reg); - } - - void analogix_dp_init_interrupt(struct analogix_dp_device *dp) - { - /* Set interrupt pin assertion polarity as high */ -- writel(INT_POL1 | INT_POL0, dp->reg_base + ANALOGIX_DP_INT_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_INT_CTL, INT_POL1 | INT_POL0); - - /* Clear pending regisers */ -- writel(0xff, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); -- writel(0x4f, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_2); -- writel(0xe0, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_3); -- writel(0xe7, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); -- writel(0x63, dp->reg_base + ANALOGIX_DP_INT_STA); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, 0xff); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_2, 0x4f); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_3, 0xe0); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_4, 0xe7); -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, 0x63); - - /* 0:mask,1: unmask */ -- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_1); -- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_2); -- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_3); -- writel(0x00, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); -- writel(0x00, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_1, 0x00); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_2, 0x00); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_3, 0x00); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, 0x00); -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, 0x00); - } - - void analogix_dp_reset(struct analogix_dp_device *dp) -@@ -130,44 +149,44 @@ void analogix_dp_reset(struct analogix_dp_device *dp) - AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N | - HDCP_FUNC_EN_N | SW_FUNC_EN_N; - -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); - - reg = SSC_FUNC_EN_N | AUX_FUNC_EN_N | - SERDES_FIFO_FUNC_EN_N | - LS_CLK_DOMAIN_FUNC_EN_N; -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); - - usleep_range(20, 30); - - analogix_dp_lane_swap(dp, 0); - -- writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); -- writel(0x40, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); -- writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -- writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, 0x0); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, 0x40); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, 0x0); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, 0x0); - -- writel(0x0, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -- writel(0x0, dp->reg_base + ANALOGIX_DP_HDCP_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, 0x0); -+ analogix_dp_write(dp, ANALOGIX_DP_HDCP_CTL, 0x0); - -- writel(0x5e, dp->reg_base + ANALOGIX_DP_HPD_DEGLITCH_L); -- writel(0x1a, dp->reg_base + ANALOGIX_DP_HPD_DEGLITCH_H); -+ analogix_dp_write(dp, ANALOGIX_DP_HPD_DEGLITCH_L, 0x5e); -+ analogix_dp_write(dp, ANALOGIX_DP_HPD_DEGLITCH_H, 0x1a); - -- writel(0x10, dp->reg_base + ANALOGIX_DP_LINK_DEBUG_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_LINK_DEBUG_CTL, 0x10); - -- writel(0x0, dp->reg_base + ANALOGIX_DP_PHY_TEST); -+ analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, 0x0); - -- writel(0x0, dp->reg_base + ANALOGIX_DP_VIDEO_FIFO_THRD); -- writel(0x20, dp->reg_base + ANALOGIX_DP_AUDIO_MARGIN); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_FIFO_THRD, 0x0); -+ analogix_dp_write(dp, ANALOGIX_DP_AUDIO_MARGIN, 0x20); - -- writel(0x4, dp->reg_base + ANALOGIX_DP_M_VID_GEN_FILTER_TH); -- writel(0x2, dp->reg_base + ANALOGIX_DP_M_AUD_GEN_FILTER_TH); -+ analogix_dp_write(dp, ANALOGIX_DP_M_VID_GEN_FILTER_TH, 0x4); -+ analogix_dp_write(dp, ANALOGIX_DP_M_AUD_GEN_FILTER_TH, 0x2); - -- writel(0x00000101, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, 0x00000101); - } - - void analogix_dp_swreset(struct analogix_dp_device *dp) - { -- writel(RESET_DP_TX, dp->reg_base + ANALOGIX_DP_TX_SW_RESET); -+ analogix_dp_write(dp, ANALOGIX_DP_TX_SW_RESET, RESET_DP_TX); - } - - void analogix_dp_config_interrupt(struct analogix_dp_device *dp) -@@ -176,19 +195,18 @@ void analogix_dp_config_interrupt(struct analogix_dp_device *dp) - - /* 0: mask, 1: unmask */ - reg = COMMON_INT_MASK_1; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_1); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_1, reg); - - reg = COMMON_INT_MASK_2; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_2); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_2, reg); - - reg = COMMON_INT_MASK_3; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_3); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_3, reg); - -- reg = COMMON_INT_MASK_4; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); -- -- reg = INT_STA_MASK; -- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); -+ if (dp->force_hpd || dp->hpd_gpiod) -+ analogix_dp_mute_hpd_interrupt(dp); -+ else -+ analogix_dp_unmute_hpd_interrupt(dp); - } - - void analogix_dp_mute_hpd_interrupt(struct analogix_dp_device *dp) -@@ -196,13 +214,13 @@ void analogix_dp_mute_hpd_interrupt(struct analogix_dp_device *dp) - u32 reg; - - /* 0: mask, 1: unmask */ -- reg = readl(dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_COMMON_INT_MASK_4); - reg &= ~COMMON_INT_MASK_4; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA_MASK); -- reg &= ~INT_STA_MASK; -- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA_MASK); -+ reg &= ~INT_HPD; -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, reg); - } - - void analogix_dp_unmute_hpd_interrupt(struct analogix_dp_device *dp) -@@ -211,17 +229,18 @@ void analogix_dp_unmute_hpd_interrupt(struct analogix_dp_device *dp) - - /* 0: mask, 1: unmask */ - reg = COMMON_INT_MASK_4; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_MASK_4); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_MASK_4, reg); - -- reg = INT_STA_MASK; -- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA_MASK); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_INT_STA_MASK); -+ reg |= INT_HPD; -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA_MASK, reg); - } - - enum pll_status analogix_dp_get_pll_lock_status(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_DEBUG_CTL); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_DEBUG_CTL); - if (reg & PLL_LOCK) - return PLL_LOCKED; - else -@@ -239,12 +258,12 @@ void analogix_dp_set_pll_power_down(struct analogix_dp_device *dp, bool enable) - mask = RK_PLL_PD; - } - -- reg = readl(dp->reg_base + pd_addr); -+ reg = analogix_dp_read(dp, pd_addr); - if (enable) - reg |= mask; - else - reg &= ~mask; -- writel(reg, dp->reg_base + pd_addr); -+ analogix_dp_write(dp, pd_addr, reg); - } - - void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, -@@ -265,52 +284,54 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, - else - mask = AUX_PD; - -- reg = readl(dp->reg_base + phy_pd_addr); -- if (enable) -+ reg = analogix_dp_read(dp, phy_pd_addr); -+ if (enable) { -+ reg &= ~(DP_INC_BG | DP_EXP_BG); - reg |= mask; -- else -+ } else { - reg &= ~mask; -- writel(reg, dp->reg_base + phy_pd_addr); -+ } -+ analogix_dp_write(dp, phy_pd_addr, reg); - break; - case CH0_BLOCK: - mask = CH0_PD; -- reg = readl(dp->reg_base + phy_pd_addr); -+ reg = analogix_dp_read(dp, phy_pd_addr); - - if (enable) - reg |= mask; - else - reg &= ~mask; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - break; - case CH1_BLOCK: - mask = CH1_PD; -- reg = readl(dp->reg_base + phy_pd_addr); -+ reg = analogix_dp_read(dp, phy_pd_addr); - - if (enable) - reg |= mask; - else - reg &= ~mask; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - break; - case CH2_BLOCK: - mask = CH2_PD; -- reg = readl(dp->reg_base + phy_pd_addr); -+ reg = analogix_dp_read(dp, phy_pd_addr); - - if (enable) - reg |= mask; - else - reg &= ~mask; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - break; - case CH3_BLOCK: - mask = CH3_PD; -- reg = readl(dp->reg_base + phy_pd_addr); -+ reg = analogix_dp_read(dp, phy_pd_addr); - - if (enable) - reg |= mask; - else - reg &= ~mask; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - break; - case ANALOG_TOTAL: - /* -@@ -323,29 +344,29 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, - else - mask = DP_PHY_PD; - -- reg = readl(dp->reg_base + phy_pd_addr); -+ reg = analogix_dp_read(dp, phy_pd_addr); - if (enable) - reg |= mask; - else - reg &= ~mask; - -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) - usleep_range(10, 15); - break; - case POWER_ALL: - if (enable) { - reg = DP_ALL_PD; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - } else { - reg = DP_ALL_PD; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - usleep_range(10, 15); - reg &= ~DP_INC_BG; -- writel(reg, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, reg); - usleep_range(10, 15); - -- writel(0x00, dp->reg_base + phy_pd_addr); -+ analogix_dp_write(dp, phy_pd_addr, 0x00); - } - break; - default: -@@ -356,36 +377,24 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp, - int analogix_dp_init_analog_func(struct analogix_dp_device *dp) - { - u32 reg; -- int timeout_loop = 0; - - analogix_dp_set_analog_power_down(dp, POWER_ALL, 0); - - reg = PLL_LOCK_CHG; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_DEBUG_CTL); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_DEBUG_CTL); - reg &= ~(F_PLL_LOCK | PLL_LOCK_CTRL); -- writel(reg, dp->reg_base + ANALOGIX_DP_DEBUG_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_DEBUG_CTL, reg); - - /* Power up PLL */ -- if (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { -- analogix_dp_set_pll_power_down(dp, 0); -- -- while (analogix_dp_get_pll_lock_status(dp) == PLL_UNLOCKED) { -- timeout_loop++; -- if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) { -- dev_err(dp->dev, "failed to get pll lock status\n"); -- return -ETIMEDOUT; -- } -- usleep_range(10, 20); -- } -- } -+ analogix_dp_set_pll_power_down(dp, 0); - - /* Enable Serdes FIFO function and Link symbol clock domain module */ -- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); - reg &= ~(SERDES_FIFO_FUNC_EN_N | LS_CLK_DOMAIN_FUNC_EN_N - | AUX_FUNC_EN_N); -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); - return 0; - } - -@@ -397,10 +406,10 @@ void analogix_dp_clear_hotplug_interrupts(struct analogix_dp_device *dp) - return; - - reg = HOTPLUG_CHG | HPD_LOST | PLUG; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_4, reg); - - reg = INT_HPD; -- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA); -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, reg); - } - - void analogix_dp_init_hpd(struct analogix_dp_device *dp) -@@ -412,45 +421,37 @@ void analogix_dp_init_hpd(struct analogix_dp_device *dp) - - analogix_dp_clear_hotplug_interrupts(dp); - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); - reg &= ~(F_HPD | HPD_CTRL); -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); - } - - void analogix_dp_force_hpd(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -- reg = (F_HPD | HPD_CTRL); -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); -+ reg |= (F_HPD | HPD_CTRL); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); - } - - enum dp_irq_type analogix_dp_get_irq_type(struct analogix_dp_device *dp) - { - u32 reg; - -- if (dp->hpd_gpiod) { -- reg = gpiod_get_value(dp->hpd_gpiod); -- if (reg) -- return DP_IRQ_TYPE_HP_CABLE_IN; -- else -- return DP_IRQ_TYPE_HP_CABLE_OUT; -- } else { -- /* Parse hotplug interrupt status register */ -- reg = readl(dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_4); -+ /* Parse hotplug interrupt status register */ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_COMMON_INT_STA_4); - -- if (reg & PLUG) -- return DP_IRQ_TYPE_HP_CABLE_IN; -+ if (reg & PLUG) -+ return DP_IRQ_TYPE_HP_CABLE_IN; - -- if (reg & HPD_LOST) -- return DP_IRQ_TYPE_HP_CABLE_OUT; -+ if (reg & HPD_LOST) -+ return DP_IRQ_TYPE_HP_CABLE_OUT; - -- if (reg & HOTPLUG_CHG) -- return DP_IRQ_TYPE_HP_CHANGE; -+ if (reg & HOTPLUG_CHG) -+ return DP_IRQ_TYPE_HP_CHANGE; - -- return DP_IRQ_TYPE_UNKNOWN; -- } -+ return DP_IRQ_TYPE_UNKNOWN; - } - - void analogix_dp_reset_aux(struct analogix_dp_device *dp) -@@ -458,9 +459,9 @@ void analogix_dp_reset_aux(struct analogix_dp_device *dp) - u32 reg; - - /* Disable AUX channel module */ -- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); - reg |= AUX_FUNC_EN_N; -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); - } - - void analogix_dp_init_aux(struct analogix_dp_device *dp) -@@ -469,7 +470,7 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp) - - /* Clear inerrupts related to AUX channel */ - reg = RPLY_RECEIV | AUX_ERR; -- writel(reg, dp->reg_base + ANALOGIX_DP_INT_STA); -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, reg); - - analogix_dp_set_analog_power_down(dp, AUX_BLOCK, true); - usleep_range(10, 11); -@@ -487,16 +488,17 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp) - reg |= AUX_HW_RETRY_COUNT_SEL(0) | - AUX_HW_RETRY_INTERVAL_600_MICROSECONDS; - -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_HW_RETRY_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_HW_RETRY_CTL, reg); - - /* Receive AUX Channel DEFER commands equal to DEFFER_COUNT*64 */ - reg = DEFER_CTRL_EN | DEFER_COUNT(1); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_DEFER_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_DEFER_CTL, reg); - - /* Enable AUX channel module */ -- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ analogix_dp_enable_sw_function(dp); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); - reg &= ~AUX_FUNC_EN_N; -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_2, reg); - } - - int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp) -@@ -507,7 +509,7 @@ int analogix_dp_get_plug_in_status(struct analogix_dp_device *dp) - if (gpiod_get_value(dp->hpd_gpiod)) - return 0; - } else { -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); - if (reg & HPD_STATUS) - return 0; - } -@@ -519,145 +521,181 @@ void analogix_dp_enable_sw_function(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); - reg &= ~SW_FUNC_EN_N; -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); - } - --int analogix_dp_start_aux_transaction(struct analogix_dp_device *dp) -+static void analogix_dp_ssc_enable(struct analogix_dp_device *dp) - { -- int reg; -- int retval = 0; -- int timeout_loop = 0; -- -- /* Enable AUX CH operation */ -- reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); -- reg |= AUX_EN; -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); -- -- /* Is AUX CH command reply received? */ -- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); -- while (!(reg & RPLY_RECEIV)) { -- timeout_loop++; -- if (DP_TIMEOUT_LOOP_COUNT < timeout_loop) { -- dev_err(dp->dev, "AUX CH command reply failed!\n"); -- return -ETIMEDOUT; -- } -- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); -- usleep_range(10, 11); -- } -- -- /* Clear interrupt source for AUX CH command reply */ -- writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA); -- -- /* Clear interrupt source for AUX CH access error */ -- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); -- if (reg & AUX_ERR) { -- writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA); -- return -EREMOTEIO; -- } -- -- /* Check AUX CH error access status */ -- reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA); -- if ((reg & AUX_STATUS_MASK) != 0) { -- dev_err(dp->dev, "AUX CH error happens: %d\n\n", -- reg & AUX_STATUS_MASK); -- return -EREMOTEIO; -- } -+ u32 reg; - -- return retval; -+ /* 4500ppm */ -+ writel(0x19, dp->reg_base + ANALOIGX_DP_SSC_REG); -+ /* -+ * To apply updated SSC parameters into SSC operation, -+ * firmware must disable and enable this bit. -+ */ -+ reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ reg |= SSC_FUNC_EN_N; -+ writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ reg &= ~SSC_FUNC_EN_N; -+ writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); - } - --int analogix_dp_write_byte_to_dpcd(struct analogix_dp_device *dp, -- unsigned int reg_addr, -- unsigned char data) -+static void analogix_dp_ssc_disable(struct analogix_dp_device *dp) - { - u32 reg; -- int i; -- int retval; -- -- for (i = 0; i < 3; i++) { -- /* Clear AUX CH data buffer */ -- reg = BUF_CLR; -- writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL); -- -- /* Select DPCD device address */ -- reg = AUX_ADDR_7_0(reg_addr); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0); -- reg = AUX_ADDR_15_8(reg_addr); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8); -- reg = AUX_ADDR_19_16(reg_addr); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16); -- -- /* Write data buffer */ -- reg = (unsigned int)data; -- writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0); - -- /* -- * Set DisplayPort transaction and write 1 byte -- * If bit 3 is 1, DisplayPort transaction. -- * If Bit 3 is 0, I2C transaction. -- */ -- reg = AUX_TX_COMM_DP_TRANSACTION | AUX_TX_COMM_WRITE; -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1); -- -- /* Start AUX transaction */ -- retval = analogix_dp_start_aux_transaction(dp); -- if (retval == 0) -- break; -- -- dev_dbg(dp->dev, "%s: Aux Transaction fail!\n", __func__); -- } -+ reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+ reg |= SSC_FUNC_EN_N; -+ writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_2); -+} - -- return retval; -+bool analogix_dp_ssc_supported(struct analogix_dp_device *dp) -+{ -+ /* Check if SSC is supported by both sides */ -+ return dp->plat_data->ssc && dp->link_train.ssc; - } - - void analogix_dp_set_link_bandwidth(struct analogix_dp_device *dp, u32 bwtype) - { -- u32 reg; -+ u32 status; -+ int ret; -+ -+ analogix_dp_write(dp, ANALOGIX_DP_LINK_BW_SET, bwtype); -+ -+ if (dp->phy) { -+ union phy_configure_opts phy_cfg = {0}; -+ -+ phy_cfg.dp.lanes = dp->link_train.lane_count; -+ phy_cfg.dp.link_rate = -+ drm_dp_bw_code_to_link_rate(dp->link_train.link_rate) / 100; -+ phy_cfg.dp.ssc = analogix_dp_ssc_supported(dp); -+ phy_cfg.dp.set_lanes = false; -+ phy_cfg.dp.set_rate = true; -+ phy_cfg.dp.set_voltages = false; -+ ret = phy_configure(dp->phy, &phy_cfg); -+ if (ret && ret != -EOPNOTSUPP) { -+ dev_err(dp->dev, "%s: phy_configure failed: %d\n", -+ __func__, ret); -+ return; -+ } -+ } else { -+ if (analogix_dp_ssc_supported(dp)) -+ analogix_dp_ssc_enable(dp); -+ else -+ analogix_dp_ssc_disable(dp); -+ } - -- reg = bwtype; -- if ((bwtype == DP_LINK_BW_2_7) || (bwtype == DP_LINK_BW_1_62)) -- writel(reg, dp->reg_base + ANALOGIX_DP_LINK_BW_SET); -+ ret = readx_poll_timeout(analogix_dp_get_pll_lock_status, dp, status, -+ status != PLL_UNLOCKED, 120, -+ 120 * DP_TIMEOUT_LOOP_COUNT); -+ if (ret) { -+ dev_err(dp->dev, "Wait for pll lock failed %d\n", ret); -+ return; -+ } - } - - void analogix_dp_get_link_bandwidth(struct analogix_dp_device *dp, u32 *bwtype) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_LINK_BW_SET); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_LINK_BW_SET); - *bwtype = reg; - } - - void analogix_dp_set_lane_count(struct analogix_dp_device *dp, u32 count) - { - u32 reg; -+ int ret; - - reg = count; -- writel(reg, dp->reg_base + ANALOGIX_DP_LANE_COUNT_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_LANE_COUNT_SET, reg); -+ -+ if (dp->phy) { -+ union phy_configure_opts phy_cfg = {0}; -+ -+ phy_cfg.dp.lanes = dp->link_train.lane_count; -+ phy_cfg.dp.set_lanes = true; -+ phy_cfg.dp.set_rate = false; -+ phy_cfg.dp.set_voltages = false; -+ ret = phy_configure(dp->phy, &phy_cfg); -+ if (ret && ret != -EOPNOTSUPP) { -+ dev_err(dp->dev, "%s: phy_configure() failed: %d\n", -+ __func__, ret); -+ return; -+ } -+ } - } - - void analogix_dp_get_lane_count(struct analogix_dp_device *dp, u32 *count) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_LANE_COUNT_SET); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_LANE_COUNT_SET); - *count = reg; - } - -+void analogix_dp_set_lane_link_training(struct analogix_dp_device *dp) -+{ -+ u8 lane; -+ int ret; -+ -+ for (lane = 0; lane < dp->link_train.lane_count; lane++) -+ analogix_dp_write(dp, -+ ANALOGIX_DP_LN0_LINK_TRAINING_CTL + 4 * lane, -+ dp->link_train.training_lane[lane]); -+ -+ if (dp->phy) { -+ union phy_configure_opts phy_cfg = {0}; -+ -+ for (lane = 0; lane < dp->link_train.lane_count; lane++) { -+ u8 training_lane = dp->link_train.training_lane[lane]; -+ u8 vs, pe; -+ -+ vs = (training_lane & DP_TRAIN_VOLTAGE_SWING_MASK) >> -+ DP_TRAIN_VOLTAGE_SWING_SHIFT; -+ pe = (training_lane & DP_TRAIN_PRE_EMPHASIS_MASK) >> -+ DP_TRAIN_PRE_EMPHASIS_SHIFT; -+ phy_cfg.dp.voltage[lane] = vs; -+ phy_cfg.dp.pre[lane] = pe; -+ } -+ -+ phy_cfg.dp.lanes = dp->link_train.lane_count; -+ phy_cfg.dp.link_rate = -+ drm_dp_bw_code_to_link_rate(dp->link_train.link_rate) / 100; -+ phy_cfg.dp.set_lanes = false; -+ phy_cfg.dp.set_rate = false; -+ phy_cfg.dp.set_voltages = true; -+ ret = phy_configure(dp->phy, &phy_cfg); -+ if (ret && ret != -EOPNOTSUPP) { -+ dev_err(dp->dev, "%s: phy_configure() failed: %d\n", -+ __func__, ret); -+ return; -+ } -+ } -+} -+ -+u32 analogix_dp_get_lane_link_training(struct analogix_dp_device *dp, u8 lane) -+{ -+ return analogix_dp_read(dp, -+ ANALOGIX_DP_LN0_LINK_TRAINING_CTL + 4 * lane); -+} -+ - void analogix_dp_enable_enhanced_mode(struct analogix_dp_device *dp, - bool enable) - { - u32 reg; - - if (enable) { -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); - reg |= ENHANCED; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); - } else { -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); - reg &= ~ENHANCED; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); - } - } - -@@ -669,144 +707,48 @@ void analogix_dp_set_training_pattern(struct analogix_dp_device *dp, - switch (pattern) { - case PRBS7: - reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_PRBS7; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - break; - case D10_2: - reg = SCRAMBLING_ENABLE | LINK_QUAL_PATTERN_SET_D10_2; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - break; - case TRAINING_PTN1: - reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN1; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - break; - case TRAINING_PTN2: - reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN2; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); -+ break; -+ case TRAINING_PTN3: -+ reg = SCRAMBLING_DISABLE | SW_TRAINING_PATTERN_SET_PTN3; -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - break; - case DP_NONE: - reg = SCRAMBLING_ENABLE | - LINK_QUAL_PATTERN_SET_DISABLE | - SW_TRAINING_PATTERN_SET_NORMAL; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - break; - default: - break; - } - } - --void analogix_dp_set_lane0_pre_emphasis(struct analogix_dp_device *dp, -- u32 level) --{ -- u32 reg; -- -- reg = readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); -- reg &= ~PRE_EMPHASIS_SET_MASK; -- reg |= level << PRE_EMPHASIS_SET_SHIFT; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane1_pre_emphasis(struct analogix_dp_device *dp, -- u32 level) --{ -- u32 reg; -- -- reg = readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); -- reg &= ~PRE_EMPHASIS_SET_MASK; -- reg |= level << PRE_EMPHASIS_SET_SHIFT; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane2_pre_emphasis(struct analogix_dp_device *dp, -- u32 level) --{ -- u32 reg; -- -- reg = readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); -- reg &= ~PRE_EMPHASIS_SET_MASK; -- reg |= level << PRE_EMPHASIS_SET_SHIFT; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane3_pre_emphasis(struct analogix_dp_device *dp, -- u32 level) --{ -- u32 reg; -- -- reg = readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); -- reg &= ~PRE_EMPHASIS_SET_MASK; -- reg |= level << PRE_EMPHASIS_SET_SHIFT; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane0_link_training(struct analogix_dp_device *dp, -- u32 training_lane) --{ -- u32 reg; -- -- reg = training_lane; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane1_link_training(struct analogix_dp_device *dp, -- u32 training_lane) --{ -- u32 reg; -- -- reg = training_lane; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane2_link_training(struct analogix_dp_device *dp, -- u32 training_lane) --{ -- u32 reg; -- -- reg = training_lane; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); --} -- --void analogix_dp_set_lane3_link_training(struct analogix_dp_device *dp, -- u32 training_lane) --{ -- u32 reg; -- -- reg = training_lane; -- writel(reg, dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); --} -- --u32 analogix_dp_get_lane0_link_training(struct analogix_dp_device *dp) --{ -- return readl(dp->reg_base + ANALOGIX_DP_LN0_LINK_TRAINING_CTL); --} -- --u32 analogix_dp_get_lane1_link_training(struct analogix_dp_device *dp) --{ -- return readl(dp->reg_base + ANALOGIX_DP_LN1_LINK_TRAINING_CTL); --} -- --u32 analogix_dp_get_lane2_link_training(struct analogix_dp_device *dp) --{ -- return readl(dp->reg_base + ANALOGIX_DP_LN2_LINK_TRAINING_CTL); --} -- --u32 analogix_dp_get_lane3_link_training(struct analogix_dp_device *dp) --{ -- return readl(dp->reg_base + ANALOGIX_DP_LN3_LINK_TRAINING_CTL); --} -- - void analogix_dp_reset_macro(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_PHY_TEST); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_PHY_TEST); - reg |= MACRO_RST; -- writel(reg, dp->reg_base + ANALOGIX_DP_PHY_TEST); -+ analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, reg); - - /* 10 us is the minimum reset time. */ - usleep_range(10, 20); - - reg &= ~MACRO_RST; -- writel(reg, dp->reg_base + ANALOGIX_DP_PHY_TEST); -+ analogix_dp_write(dp, ANALOGIX_DP_PHY_TEST, reg); - } - - void analogix_dp_init_video(struct analogix_dp_device *dp) -@@ -814,19 +756,19 @@ void analogix_dp_init_video(struct analogix_dp_device *dp) - u32 reg; - - reg = VSYNC_DET | VID_FORMAT_CHG | VID_CLK_CHG; -- writel(reg, dp->reg_base + ANALOGIX_DP_COMMON_INT_STA_1); -+ analogix_dp_write(dp, ANALOGIX_DP_COMMON_INT_STA_1, reg); - - reg = 0x0; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, reg); - - reg = CHA_CRI(4) | CHA_CTRL; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, reg); - - reg = 0x0; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); - - reg = VID_HRES_TH(2) | VID_VRES_TH(0); -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_8); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_8, reg); - } - - void analogix_dp_set_video_color_format(struct analogix_dp_device *dp) -@@ -837,36 +779,36 @@ void analogix_dp_set_video_color_format(struct analogix_dp_device *dp) - reg = (dp->video_info.dynamic_range << IN_D_RANGE_SHIFT) | - (dp->video_info.color_depth << IN_BPC_SHIFT) | - (dp->video_info.color_space << IN_COLOR_F_SHIFT); -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_2); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_2, reg); - - /* Set Input Color YCbCr Coefficients to ITU601 or ITU709 */ -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_3); - reg &= ~IN_YC_COEFFI_MASK; - if (dp->video_info.ycbcr_coeff) - reg |= IN_YC_COEFFI_ITU709; - else - reg |= IN_YC_COEFFI_ITU601; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_3, reg); - } - - int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_1); -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_1, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_1); - - if (!(reg & DET_STA)) { - dev_dbg(dp->dev, "Input stream clock not detected.\n"); - return -EINVAL; - } - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2); -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_2); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_2); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_2, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_2); - dev_dbg(dp->dev, "wait SYS_CTL_2.\n"); - - if (reg & CHA_STA) { -@@ -884,30 +826,30 @@ void analogix_dp_set_video_cr_mn(struct analogix_dp_device *dp, - u32 reg; - - if (type == REGISTER_M) { -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); - reg |= FIX_M_VID; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); - reg = m_value & 0xff; -- writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_0); -+ analogix_dp_write(dp, ANALOGIX_DP_M_VID_0, reg); - reg = (m_value >> 8) & 0xff; -- writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_1); -+ analogix_dp_write(dp, ANALOGIX_DP_M_VID_1, reg); - reg = (m_value >> 16) & 0xff; -- writel(reg, dp->reg_base + ANALOGIX_DP_M_VID_2); -+ analogix_dp_write(dp, ANALOGIX_DP_M_VID_2, reg); - - reg = n_value & 0xff; -- writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_0); -+ analogix_dp_write(dp, ANALOGIX_DP_N_VID_0, reg); - reg = (n_value >> 8) & 0xff; -- writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_1); -+ analogix_dp_write(dp, ANALOGIX_DP_N_VID_1, reg); - reg = (n_value >> 16) & 0xff; -- writel(reg, dp->reg_base + ANALOGIX_DP_N_VID_2); -+ analogix_dp_write(dp, ANALOGIX_DP_N_VID_2, reg); - } else { -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); - reg &= ~FIX_M_VID; -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_4); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); - -- writel(0x00, dp->reg_base + ANALOGIX_DP_N_VID_0); -- writel(0x80, dp->reg_base + ANALOGIX_DP_N_VID_1); -- writel(0x00, dp->reg_base + ANALOGIX_DP_N_VID_2); -+ analogix_dp_write(dp, ANALOGIX_DP_N_VID_0, 0x00); -+ analogix_dp_write(dp, ANALOGIX_DP_N_VID_1, 0x80); -+ analogix_dp_write(dp, ANALOGIX_DP_N_VID_2, 0x00); - } - } - -@@ -916,13 +858,13 @@ void analogix_dp_set_video_timing_mode(struct analogix_dp_device *dp, u32 type) - u32 reg; - - if (type == VIDEO_TIMING_FROM_CAPTURE) { -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); - reg &= ~FORMAT_SEL; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - } else { -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); - reg |= FORMAT_SEL; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - } - } - -@@ -931,15 +873,15 @@ void analogix_dp_enable_video_master(struct analogix_dp_device *dp, bool enable) - u32 reg; - - if (enable) { -- reg = readl(dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SOC_GENERAL_CTL); - reg &= ~VIDEO_MODE_MASK; - reg |= VIDEO_MASTER_MODE_EN | VIDEO_MODE_MASTER_MODE; -- writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); - } else { -- reg = readl(dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SOC_GENERAL_CTL); - reg &= ~VIDEO_MODE_MASK; - reg |= VIDEO_MODE_SLAVE_MODE; -- writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); - } - } - -@@ -947,19 +889,19 @@ void analogix_dp_start_video(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_1); - reg |= VIDEO_EN; -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_1, reg); - } - - int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -- writel(reg, dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_3, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_3); - if (!(reg & STRM_VALID)) { - dev_dbg(dp->dev, "Input video stream is not detected.\n"); - return -EINVAL; -@@ -972,55 +914,55 @@ void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); - if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) { - reg &= ~(RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N); - } else { - reg &= ~(MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N); - reg |= MASTER_VID_FUNC_EN_N; - } -- writel(reg, dp->reg_base + ANALOGIX_DP_FUNC_EN_1); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); - reg &= ~INTERACE_SCAN_CFG; - reg |= (dp->video_info.interlaced << 2); -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); - reg &= ~VSYNC_POLARITY_CFG; - reg |= (dp->video_info.v_sync_polarity << 1); -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - -- reg = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); - reg &= ~HSYNC_POLARITY_CFG; - reg |= (dp->video_info.h_sync_polarity << 0); -- writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_10); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); - - reg = AUDIO_MODE_SPDIF_MODE | VIDEO_MODE_SLAVE_MODE; -- writel(reg, dp->reg_base + ANALOGIX_DP_SOC_GENERAL_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_SOC_GENERAL_CTL, reg); - } - - void analogix_dp_enable_scrambling(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_TRAINING_PTN_SET); - reg &= ~SCRAMBLING_DISABLE; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - } - - void analogix_dp_disable_scrambling(struct analogix_dp_device *dp) - { - u32 reg; - -- reg = readl(dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_TRAINING_PTN_SET); - reg |= SCRAMBLING_DISABLE; -- writel(reg, dp->reg_base + ANALOGIX_DP_TRAINING_PTN_SET); -+ analogix_dp_write(dp, ANALOGIX_DP_TRAINING_PTN_SET, reg); - } - - void analogix_dp_enable_psr_crc(struct analogix_dp_device *dp) - { -- writel(PSR_VID_CRC_ENABLE, dp->reg_base + ANALOGIX_DP_CRC_CON); -+ analogix_dp_write(dp, ANALOGIX_DP_CRC_CON, PSR_VID_CRC_ENABLE); - } - - static ssize_t analogix_dp_get_psr_status(struct analogix_dp_device *dp) -@@ -1044,44 +986,44 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, - ssize_t psr_status; - - /* don't send info frame */ -- val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -+ val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); - val &= ~IF_EN; -- writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); - - /* configure single frame update mode */ -- writel(PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE, -- dp->reg_base + ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL); -+ analogix_dp_write(dp, ANALOGIX_DP_PSR_FRAME_UPDATE_CTRL, -+ PSR_FRAME_UP_TYPE_BURST | PSR_CRC_SEL_HARDWARE); - - /* configure VSC HB0~HB3 */ -- writel(vsc->sdp_header.HB0, dp->reg_base + ANALOGIX_DP_SPD_HB0); -- writel(vsc->sdp_header.HB1, dp->reg_base + ANALOGIX_DP_SPD_HB1); -- writel(vsc->sdp_header.HB2, dp->reg_base + ANALOGIX_DP_SPD_HB2); -- writel(vsc->sdp_header.HB3, dp->reg_base + ANALOGIX_DP_SPD_HB3); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB0, vsc->sdp_header.HB0); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB1, vsc->sdp_header.HB1); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB2, vsc->sdp_header.HB2); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_HB3, vsc->sdp_header.HB3); - - /* configure reused VSC PB0~PB3, magic number from vendor */ -- writel(0x00, dp->reg_base + ANALOGIX_DP_SPD_PB0); -- writel(0x16, dp->reg_base + ANALOGIX_DP_SPD_PB1); -- writel(0xCE, dp->reg_base + ANALOGIX_DP_SPD_PB2); -- writel(0x5D, dp->reg_base + ANALOGIX_DP_SPD_PB3); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB0, 0x00); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB1, 0x16); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB2, 0xCE); -+ analogix_dp_write(dp, ANALOGIX_DP_SPD_PB3, 0x5D); - - /* configure DB0 / DB1 values */ -- writel(vsc->db[0], dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB0); -- writel(vsc->db[1], dp->reg_base + ANALOGIX_DP_VSC_SHADOW_DB1); -+ analogix_dp_write(dp, ANALOGIX_DP_VSC_SHADOW_DB0, vsc->db[0]); -+ analogix_dp_write(dp, ANALOGIX_DP_VSC_SHADOW_DB1, vsc->db[1]); - - /* set reuse spd inforframe */ -- val = readl(dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); -+ val = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_3); - val |= REUSE_SPD_EN; -- writel(val, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3); -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_3, val); - - /* mark info frame update */ -- val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -+ val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); - val = (val | IF_UP) & ~IF_EN; -- writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); - - /* send info frame */ -- val = readl(dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -+ val = analogix_dp_read(dp, ANALOGIX_DP_PKT_SEND_CTL); - val |= IF_EN; -- writel(val, dp->reg_base + ANALOGIX_DP_PKT_SEND_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_PKT_SEND_CTL, val); - - if (!blocking) - return 0; -@@ -1098,11 +1040,43 @@ int analogix_dp_send_psr_spd(struct analogix_dp_device *dp, - return 0; - } - -+void analogix_dp_phy_power_on(struct analogix_dp_device *dp) -+{ -+ if (dp->phy_enabled) -+ return; -+ -+ phy_set_mode(dp->phy, PHY_MODE_DP); -+ phy_power_on(dp->phy); -+ -+ dp->phy_enabled = true; -+} -+ -+void analogix_dp_phy_power_off(struct analogix_dp_device *dp) -+{ -+ if (!dp->phy_enabled) -+ return; -+ -+ phy_power_off(dp->phy); -+ -+ dp->phy_enabled = false; -+} -+ -+enum { -+ AUX_STATUS_OK, -+ AUX_STATUS_NACK_ERROR, -+ AUX_STATUS_TIMEOUT_ERROR, -+ AUX_STATUS_UNKNOWN_ERROR, -+ AUX_STATUS_MUCH_DEFER_ERROR, -+ AUX_STATUS_TX_SHORT_ERROR, -+ AUX_STATUS_RX_SHORT_ERROR, -+ AUX_STATUS_NACK_WITHOUT_M_ERROR, -+ AUX_STATUS_I2C_NACK_ERROR -+}; -+ - ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - struct drm_dp_aux_msg *msg) - { - u32 reg; -- u32 status_reg; - u8 *buffer = msg->buffer; - unsigned int i; - int num_transferred = 0; -@@ -1112,9 +1086,15 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - if (WARN_ON(msg->size > 16)) - return -E2BIG; - -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_2); -+ if (reg & AUX_FUNC_EN_N) { -+ analogix_dp_phy_power_on(dp); -+ analogix_dp_init_aux(dp); -+ } -+ - /* Clear AUX CH data buffer */ - reg = BUF_CLR; -- writel(reg, dp->reg_base + ANALOGIX_DP_BUFFER_DATA_CTL); -+ analogix_dp_write(dp, ANALOGIX_DP_BUFFER_DATA_CTL, reg); - - switch (msg->request & ~DP_AUX_I2C_MOT) { - case DP_AUX_I2C_WRITE: -@@ -1142,21 +1122,21 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - } - - reg |= AUX_LENGTH(msg->size); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_1); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_CTL_1, reg); - - /* Select DPCD device address */ - reg = AUX_ADDR_7_0(msg->address); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_7_0); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_7_0, reg); - reg = AUX_ADDR_15_8(msg->address); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_15_8); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_15_8, reg); - reg = AUX_ADDR_19_16(msg->address); -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_ADDR_19_16); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_ADDR_19_16, reg); - - if (!(msg->request & DP_AUX_I2C_READ)) { - for (i = 0; i < msg->size; i++) { - reg = buffer[i]; -- writel(reg, dp->reg_base + ANALOGIX_DP_BUF_DATA_0 + -- 4 * i); -+ analogix_dp_write(dp, ANALOGIX_DP_BUF_DATA_0 + 4 * i, -+ reg); - num_transferred++; - } - } -@@ -1168,7 +1148,7 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - if (msg->size < 1) - reg |= ADDR_ONLY; - -- writel(reg, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2); -+ analogix_dp_write(dp, ANALOGIX_DP_AUX_CH_CTL_2, reg); - - ret = readx_poll_timeout(readl, dp->reg_base + ANALOGIX_DP_AUX_CH_CTL_2, - reg, !(reg & AUX_EN), 25, 500 * 1000); -@@ -1187,30 +1167,31 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - } - - /* Clear interrupt source for AUX CH command reply */ -- writel(RPLY_RECEIV, dp->reg_base + ANALOGIX_DP_INT_STA); -+ analogix_dp_write(dp, ANALOGIX_DP_INT_STA, RPLY_RECEIV); - -- /* Clear interrupt source for AUX CH access error */ -- reg = readl(dp->reg_base + ANALOGIX_DP_INT_STA); -- status_reg = readl(dp->reg_base + ANALOGIX_DP_AUX_CH_STA); -- if ((reg & AUX_ERR) || (status_reg & AUX_STATUS_MASK)) { -- writel(AUX_ERR, dp->reg_base + ANALOGIX_DP_INT_STA); -- -- dev_warn(dp->dev, "AUX CH error happened: %#x (%d)\n", -- status_reg & AUX_STATUS_MASK, !!(reg & AUX_ERR)); -- goto aux_error; -- } -+ reg = analogix_dp_read(dp, ANALOGIX_DP_AUX_CH_STA); -+ if ((reg & AUX_STATUS_MASK) == AUX_STATUS_TIMEOUT_ERROR) -+ return -ETIMEDOUT; - - if (msg->request & DP_AUX_I2C_READ) { -+ size_t buf_data_count; -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_BUFFER_DATA_CTL); -+ buf_data_count = BUF_DATA_COUNT(reg); -+ -+ if (buf_data_count != msg->size) -+ return -EBUSY; -+ - for (i = 0; i < msg->size; i++) { -- reg = readl(dp->reg_base + ANALOGIX_DP_BUF_DATA_0 + -- 4 * i); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_BUF_DATA_0 + -+ 4 * i); - buffer[i] = (unsigned char)reg; - num_transferred++; - } - } - - /* Check if Rx sends defer */ -- reg = readl(dp->reg_base + ANALOGIX_DP_AUX_RX_COMM); -+ reg = analogix_dp_read(dp, ANALOGIX_DP_AUX_RX_COMM); - if (reg == AUX_RX_COMM_AUX_DEFER) - msg->reply = DP_AUX_NATIVE_REPLY_DEFER; - else if (reg == AUX_RX_COMM_I2C_DEFER) -@@ -1222,7 +1203,7 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - (msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ) - msg->reply = DP_AUX_NATIVE_REPLY_ACK; - -- return num_transferred > 0 ? num_transferred : -EBUSY; -+ return (num_transferred == msg->size) ? num_transferred : -EBUSY; - - aux_error: - /* if aux err happen, reset aux */ -@@ -1230,3 +1211,119 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp, - - return -EREMOTEIO; - } -+ -+void analogix_dp_set_video_format(struct analogix_dp_device *dp) -+{ -+ struct video_info *video = &dp->video_info; -+ const struct drm_display_mode *mode = &video->mode; -+ unsigned int hsw, hfp, hbp, vsw, vfp, vbp; -+ -+ hsw = mode->hsync_end - mode->hsync_start; -+ hfp = mode->hsync_start - mode->hdisplay; -+ hbp = mode->htotal - mode->hsync_end; -+ vsw = mode->vsync_end - mode->vsync_start; -+ vfp = mode->vsync_start - mode->vdisplay; -+ vbp = mode->vtotal - mode->vsync_end; -+ -+ /* Set Video Format Parameters */ -+ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_LINE_CFG_L, -+ TOTAL_LINE_CFG_L(mode->vtotal)); -+ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_LINE_CFG_H, -+ TOTAL_LINE_CFG_H(mode->vtotal >> 8)); -+ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_LINE_CFG_L, -+ ACTIVE_LINE_CFG_L(mode->vdisplay)); -+ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_LINE_CFG_H, -+ ACTIVE_LINE_CFG_H(mode->vdisplay >> 8)); -+ analogix_dp_write(dp, ANALOGIX_DP_V_F_PORCH_CFG, -+ V_F_PORCH_CFG(vfp)); -+ analogix_dp_write(dp, ANALOGIX_DP_V_SYNC_WIDTH_CFG, -+ V_SYNC_WIDTH_CFG(vsw)); -+ analogix_dp_write(dp, ANALOGIX_DP_V_B_PORCH_CFG, -+ V_B_PORCH_CFG(vbp)); -+ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_PIXEL_CFG_L, -+ TOTAL_PIXEL_CFG_L(mode->htotal)); -+ analogix_dp_write(dp, ANALOGIX_DP_TOTAL_PIXEL_CFG_H, -+ TOTAL_PIXEL_CFG_H(mode->htotal >> 8)); -+ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_PIXEL_CFG_L, -+ ACTIVE_PIXEL_CFG_L(mode->hdisplay)); -+ analogix_dp_write(dp, ANALOGIX_DP_ACTIVE_PIXEL_CFG_H, -+ ACTIVE_PIXEL_CFG_H(mode->hdisplay >> 8)); -+ analogix_dp_write(dp, ANALOGIX_DP_H_F_PORCH_CFG_L, -+ H_F_PORCH_CFG_L(hfp)); -+ analogix_dp_write(dp, ANALOGIX_DP_H_F_PORCH_CFG_H, -+ H_F_PORCH_CFG_H(hfp >> 8)); -+ analogix_dp_write(dp, ANALOGIX_DP_H_SYNC_CFG_L, -+ H_SYNC_CFG_L(hsw)); -+ analogix_dp_write(dp, ANALOGIX_DP_H_SYNC_CFG_H, -+ H_SYNC_CFG_H(hsw >> 8)); -+ analogix_dp_write(dp, ANALOGIX_DP_H_B_PORCH_CFG_L, -+ H_B_PORCH_CFG_L(hbp)); -+ analogix_dp_write(dp, ANALOGIX_DP_H_B_PORCH_CFG_H, -+ H_B_PORCH_CFG_H(hbp >> 8)); -+} -+ -+void analogix_dp_video_bist_enable(struct analogix_dp_device *dp) -+{ -+ u32 reg; -+ -+ /* Enable Video BIST */ -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_4, BIST_EN); -+ -+ /* -+ * Note that if BIST_EN is set to 1, F_SEL must be cleared to 0 -+ * although video format information comes from registers set by user. -+ */ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_VIDEO_CTL_10); -+ reg &= ~FORMAT_SEL; -+ analogix_dp_write(dp, ANALOGIX_DP_VIDEO_CTL_10, reg); -+} -+ -+void analogix_dp_audio_config_i2s(struct analogix_dp_device *dp) -+{ -+ u32 reg; -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); -+ reg &= ~FIX_M_AUD; -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_I2S_CTRL); -+ reg |= I2S_EN; -+ analogix_dp_write(dp, ANALOGIX_DP_I2S_CTRL, reg); -+} -+ -+void analogix_dp_audio_config_spdif(struct analogix_dp_device *dp) -+{ -+ u32 reg; -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SYS_CTL_4); -+ reg &= ~FIX_M_AUD; -+ analogix_dp_write(dp, ANALOGIX_DP_SYS_CTL_4, reg); -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_SPDIF_AUDIO_CTL_0); -+ reg |= AUD_SPDIF_EN; -+ analogix_dp_write(dp, ANALOGIX_DP_SPDIF_AUDIO_CTL_0, reg); -+} -+ -+void analogix_dp_audio_enable(struct analogix_dp_device *dp) -+{ -+ u32 reg; -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); -+ reg &= ~(AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N); -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_AUD_CTL); -+ reg |= MISC_CTRL_RESET | DP_AUDIO_EN; -+ analogix_dp_write(dp, ANALOGIX_DP_AUD_CTL, reg); -+} -+ -+void analogix_dp_audio_disable(struct analogix_dp_device *dp) -+{ -+ u32 reg; -+ -+ analogix_dp_write(dp, ANALOGIX_DP_AUD_CTL, 0); -+ -+ reg = analogix_dp_read(dp, ANALOGIX_DP_FUNC_EN_1); -+ reg |= AUD_FIFO_FUNC_EN_N | AUD_FUNC_EN_N; -+ analogix_dp_write(dp, ANALOGIX_DP_FUNC_EN_1, reg); -+} -diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h -index e284ee8da..df88f1ad0 100644 ---- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h -+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h -@@ -15,9 +15,27 @@ - #define ANALOGIX_DP_VIDEO_CTL_1 0x20 - #define ANALOGIX_DP_VIDEO_CTL_2 0x24 - #define ANALOGIX_DP_VIDEO_CTL_3 0x28 -+#define ANALOGIX_DP_VIDEO_CTL_4 0x2C - - #define ANALOGIX_DP_VIDEO_CTL_8 0x3C - #define ANALOGIX_DP_VIDEO_CTL_10 0x44 -+#define ANALOGIX_DP_TOTAL_LINE_CFG_L 0x48 -+#define ANALOGIX_DP_TOTAL_LINE_CFG_H 0x4C -+#define ANALOGIX_DP_ACTIVE_LINE_CFG_L 0x50 -+#define ANALOGIX_DP_ACTIVE_LINE_CFG_H 0x54 -+#define ANALOGIX_DP_V_F_PORCH_CFG 0x58 -+#define ANALOGIX_DP_V_SYNC_WIDTH_CFG 0x5C -+#define ANALOGIX_DP_V_B_PORCH_CFG 0x60 -+#define ANALOGIX_DP_TOTAL_PIXEL_CFG_L 0x64 -+#define ANALOGIX_DP_TOTAL_PIXEL_CFG_H 0x68 -+#define ANALOGIX_DP_ACTIVE_PIXEL_CFG_L 0x6C -+#define ANALOGIX_DP_ACTIVE_PIXEL_CFG_H 0x70 -+#define ANALOGIX_DP_H_F_PORCH_CFG_L 0x74 -+#define ANALOGIX_DP_H_F_PORCH_CFG_H 0x78 -+#define ANALOGIX_DP_H_SYNC_CFG_L 0x7C -+#define ANALOGIX_DP_H_SYNC_CFG_H 0x80 -+#define ANALOGIX_DP_H_B_PORCH_CFG_L 0x84 -+#define ANALOGIX_DP_H_B_PORCH_CFG_H 0x88 - - #define ANALOGIX_DP_SPDIF_AUDIO_CTL_0 0xD8 - -@@ -27,6 +45,8 @@ - #define ANALOGIX_DP_PLL_REG_4 0x9ec - #define ANALOGIX_DP_PLL_REG_5 0xa00 - -+#define ANALOIGX_DP_SSC_REG 0x104 -+#define ANALOGIX_DP_BIAS 0x124 - #define ANALOGIX_DP_PD 0x12c - - #define ANALOGIX_DP_IF_TYPE 0x244 -@@ -70,7 +90,7 @@ - #define ANALOGIX_DP_SYS_CTL_2 0x604 - #define ANALOGIX_DP_SYS_CTL_3 0x608 - #define ANALOGIX_DP_SYS_CTL_4 0x60C -- -+#define ANALOGIX_DP_AUD_CTL 0x618 - #define ANALOGIX_DP_PKT_SEND_CTL 0x640 - #define ANALOGIX_DP_HDCP_CTL 0x648 - -@@ -116,8 +136,9 @@ - #define ANALOGIX_DP_BUF_DATA_0 0x7C0 - - #define ANALOGIX_DP_SOC_GENERAL_CTL 0x800 -- -+#define ANALOGIX_DP_AUD_CHANNEL_CTL 0x834 - #define ANALOGIX_DP_CRC_CON 0x890 -+#define ANALOGIX_DP_I2S_CTRL 0x9C8 - - /* ANALOGIX_DP_TX_SW_RESET */ - #define RESET_DP_TX (0x1 << 0) -@@ -171,6 +192,11 @@ - #define VID_CHK_UPDATE_TYPE_0 (0x0 << 4) - #define REUSE_SPD_EN (0x1 << 3) - -+/* ANALOGIX_DP_VIDEO_CTL_4 */ -+#define BIST_EN (0x1 << 3) -+#define BIST_WIDTH(x) (((x) & 0x1) << 2) -+#define BIST_TYPE(x) (((x) & 0x3) << 0) -+ - /* ANALOGIX_DP_VIDEO_CTL_8 */ - #define VID_HRES_TH(x) (((x) & 0xf) << 4) - #define VID_VRES_TH(x) (((x) & 0xf) << 0) -@@ -181,6 +207,60 @@ - #define VSYNC_POLARITY_CFG (0x1 << 1) - #define HSYNC_POLARITY_CFG (0x1 << 0) - -+/* ANALOGIX_DP_TOTAL_LINE_CFG_L */ -+#define TOTAL_LINE_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_TOTAL_LINE_CFG_H */ -+#define TOTAL_LINE_CFG_H(x) (((x) & 0xf) << 0) -+ -+/* ANALOGIX_DP_ACTIVE_LINE_CFG_L */ -+#define ACTIVE_LINE_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_ACTIVE_LINE_CFG_H */ -+#define ACTIVE_LINE_CFG_H(x) (((x) & 0xf) << 0) -+ -+/* ANALOGIX_DP_V_F_PORCH_CFG */ -+#define V_F_PORCH_CFG(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_V_SYNC_WIDTH_CFG */ -+#define V_SYNC_WIDTH_CFG(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_V_B_PORCH_CFG */ -+#define V_B_PORCH_CFG(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_TOTAL_PIXEL_CFG_L */ -+#define TOTAL_PIXEL_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_TOTAL_PIXEL_CFG_H */ -+#define TOTAL_PIXEL_CFG_H(x) (((x) & 0x3f) << 0) -+ -+/* ANALOGIX_DP_ACTIVE_PIXEL_CFG_L */ -+#define ACTIVE_PIXEL_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_ACTIVE_PIXEL_CFG_H */ -+#define ACTIVE_PIXEL_CFG_H(x) (((x) & 0x3f) << 0) -+ -+/* ANALOGIX_DP_H_F_PORCH_CFG_L */ -+#define H_F_PORCH_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_H_F_PORCH_CFG_H */ -+#define H_F_PORCH_CFG_H(x) (((x) & 0xf) << 0) -+ -+/* ANALOGIX_DP_H_SYNC_CFG_L */ -+#define H_SYNC_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_H_SYNC_CFG_H */ -+#define H_SYNC_CFG_H(x) (((x) & 0xf) << 0) -+ -+/* ANALOGIX_DP_H_B_PORCH_CFG_L */ -+#define H_B_PORCH_CFG_L(x) (((x) & 0xff) << 0) -+ -+/* ANALOGIX_DP_H_B_PORCH_CFG_H */ -+#define H_B_PORCH_CFG_H(x) (((x) & 0xf) << 0) -+ -+/* ANALOGIX_DP_SPDIF_AUDIO_CTL_0 */ -+#define AUD_SPDIF_EN (0x1 << 7) -+ - /* ANALOGIX_DP_PLL_REG_1 */ - #define REF_CLK_24M (0x1 << 0) - #define REF_CLK_27M (0x0 << 0) -@@ -309,6 +389,10 @@ - #define FIX_M_VID (0x1 << 2) - #define M_VID_UPDATE_CTRL (0x3 << 0) - -+/* ANALOGIX_DP_AUD_CTL */ -+#define MISC_CTRL_RESET (0x1 << 4) -+#define DP_AUDIO_EN (0x1 << 0) -+ - /* ANALOGIX_DP_TRAINING_PTN_SET */ - #define SCRAMBLER_TYPE (0x1 << 9) - #define HW_LINK_TRAINING_PATTERN (0x1 << 8) -@@ -319,6 +403,7 @@ - #define LINK_QUAL_PATTERN_SET_D10_2 (0x1 << 2) - #define LINK_QUAL_PATTERN_SET_DISABLE (0x0 << 2) - #define SW_TRAINING_PATTERN_SET_MASK (0x3 << 0) -+#define SW_TRAINING_PATTERN_SET_PTN3 (0x3 << 0) - #define SW_TRAINING_PATTERN_SET_PTN2 (0x2 << 0) - #define SW_TRAINING_PATTERN_SET_PTN1 (0x1 << 0) - #define SW_TRAINING_PATTERN_SET_NORMAL (0x0 << 0) -@@ -406,6 +491,11 @@ - #define VIDEO_MODE_SLAVE_MODE (0x1 << 0) - #define VIDEO_MODE_MASTER_MODE (0x0 << 0) - -+/* ANALOGIX_DP_AUD_CHANNEL_CTL */ -+#define AUD_CHANNEL_COUNT_6 (0x5 << 0) -+#define AUD_CHANNEL_COUNT_4 (0x3 << 0) -+#define AUD_CHANNEL_COUNT_2 (0x1 << 0) -+ - /* ANALOGIX_DP_PKT_SEND_CTL */ - #define IF_UP (0x1 << 4) - #define IF_EN (0x1 << 0) -@@ -414,4 +504,7 @@ - #define PSR_VID_CRC_FLUSH (0x1 << 2) - #define PSR_VID_CRC_ENABLE (0x1 << 0) - -+/* ANALOGIX_DP_I2S_CTRL */ -+#define I2S_EN (0x1 << 4) -+ - #endif /* _ANALOGIX_DP_REG_H */ -diff --git a/drivers/gpu/drm/bridge/synopsys/Makefile b/drivers/gpu/drm/bridge/synopsys/Makefile -index 91d746ad5..ea11fa1d4 100644 ---- a/drivers/gpu/drm/bridge/synopsys/Makefile -+++ b/drivers/gpu/drm/bridge/synopsys/Makefile -@@ -1,7 +1,8 @@ - # SPDX-License-Identifier: GPL-2.0-only --obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o -+obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o dw-hdmi-hdcp.o \ -+ dw-hdmi-qp.o - obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o --obj-$(CONFIG_DRM_DW_HDMI_I2S_AUDIO) += dw-hdmi-i2s-audio.o -+obj-$(CONFIG_DRM_DW_HDMI_I2S_AUDIO) += dw-hdmi-i2s-audio.o dw-hdmi-qp-i2s-audio.o - obj-$(CONFIG_DRM_DW_HDMI_CEC) += dw-hdmi-cec.o - - obj-$(CONFIG_DRM_DW_MIPI_DSI) += dw-mipi-dsi.o -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c -index 70ab4fbdc..48fc36d56 100644 ---- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c -@@ -12,6 +12,7 @@ - #include - - #include -+#include - - #include - #include -@@ -262,6 +263,8 @@ static int dw_hdmi_cec_probe(struct platform_device *pdev) - if (IS_ERR(cec->adap)) - return PTR_ERR(cec->adap); - -+ dw_hdmi_set_cec_adap(cec->hdmi, cec->adap); -+ - /* override the module pointer */ - cec->adap->owner = THIS_MODULE; - -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c -new file mode 100755 -index 000000000..24aab5043 ---- /dev/null -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.c -@@ -0,0 +1,748 @@ -+/* -+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd -+ * Author Huicong Xu -+ * -+ * This software is licensed under the terms of the GNU General Public -+ * License version 2, as published by the Free Software Foundation, and -+ * may be copied, distributed, and modified under those terms. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "dw-hdmi.h" -+#include "dw-hdmi-hdcp.h" -+ -+#define HDCP_KEY_SIZE 308 -+#define HDCP_KEY_SEED_SIZE 2 -+ -+#define KSV_LEN 5 -+#define HEADER 10 -+#define SHAMAX 20 -+ -+#define MAX_DOWNSTREAM_DEVICE_NUM 5 -+#define DPK_WR_OK_TIMEOUT_US 30000 -+#define HDMI_HDCP1X_ID 5 -+ -+/* HDCP Registers */ -+#define HDMI_HDCPREG_RMCTL 0x780e -+#define HDMI_HDCPREG_RMSTS 0x780f -+#define HDMI_HDCPREG_SEED0 0x7810 -+#define HDMI_HDCPREG_SEED1 0x7811 -+#define HDMI_HDCPREG_DPK0 0x7812 -+#define HDMI_HDCPREG_DPK1 0x7813 -+#define HDMI_HDCPREG_DPK2 0x7814 -+#define HDMI_HDCPREG_DPK3 0x7815 -+#define HDMI_HDCPREG_DPK4 0x7816 -+#define HDMI_HDCPREG_DPK5 0x7817 -+#define HDMI_HDCPREG_DPK6 0x7818 -+#define HDMI_HDCP2REG_CTRL 0x7904 -+#define HDMI_HDCP2REG_MASK 0x790c -+#define HDMI_HDCP2REG_MUTE 0x790e -+ -+enum dw_hdmi_hdcp_state { -+ DW_HDCP_DISABLED, -+ DW_HDCP_AUTH_START, -+ DW_HDCP_AUTH_SUCCESS, -+ DW_HDCP_AUTH_FAIL, -+}; -+ -+enum { -+ DW_HDMI_HDCP_KSV_LEN = 8, -+ DW_HDMI_HDCP_SHA_LEN = 20, -+ DW_HDMI_HDCP_DPK_LEN = 280, -+ DW_HDMI_HDCP_KEY_LEN = 308, -+ DW_HDMI_HDCP_SEED_LEN = 2, -+}; -+ -+enum { -+ HDMI_MC_CLKDIS_HDCPCLK_MASK = 0x40, -+ HDMI_MC_CLKDIS_HDCPCLK_ENABLE = 0x00, -+ -+ HDMI_A_SRMCTRL_SHA1_FAIL_MASK = 0X08, -+ HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE = 0X00, -+ HDMI_A_SRMCTRL_SHA1_FAIL_ENABLE = 0X08, -+ -+ HDMI_A_SRMCTRL_KSV_UPDATE_MASK = 0X04, -+ HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE = 0X00, -+ HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE = 0X04, -+ -+ HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK = 0X01, -+ HDMI_A_SRMCTRL_KSV_MEM_REQ_DISABLE = 0X00, -+ HDMI_A_SRMCTRL_KSV_MEM_REQ_ENABLE = 0X01, -+ -+ HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK = 0X02, -+ HDMI_A_SRMCTRL_KSV_MEM_ACCESS_DISABLE = 0X00, -+ HDMI_A_SRMCTRL_KSV_MEM_ACCESS_ENABLE = 0X02, -+ -+ HDMI_A_SRM_BASE_MAX_DEVS_EXCEEDED = 0x80, -+ HDMI_A_SRM_BASE_DEVICE_COUNT = 0x7f, -+ -+ HDMI_A_SRM_BASE_MAX_CASCADE_EXCEEDED = 0x08, -+ -+ HDMI_A_APIINTSTAT_KSVSHA1_CALC_INT = 0x02, -+ -+ /* HDCPREG_RMSTS field values */ -+ DPK_WR_OK_STS = 0x40, -+ -+ HDMI_A_HDCP22_MASK = 0x40, -+ -+ HDMI_HDCP2_OVR_EN_MASK = 0x02, -+ HDMI_HDCP2_OVR_ENABLE = 0x02, -+ HDMI_HDCP2_OVR_DISABLE = 0x00, -+ -+ HDMI_HDCP2_FORCE_MASK = 0x04, -+ HDMI_HDCP2_FORCE_ENABLE = 0x04, -+ HDMI_HDCP2_FORCE_DISABLE = 0x00, -+}; -+ -+struct sha_t { -+ u8 mlength[8]; -+ u8 mblock[64]; -+ int mindex; -+ int mcomputed; -+ int mcorrupted; -+ unsigned int mdigest[5]; -+}; -+ -+static struct dw_hdcp *g_hdcp; -+ -+static inline unsigned int shacircularshift(unsigned int bits, -+ unsigned int word) -+{ -+ return (((word << bits) & 0xFFFFFFFF) | (word >> (32 - bits))); -+} -+ -+static void hdcp_modb(struct dw_hdcp *hdcp, u8 data, u8 mask, unsigned int reg) -+{ -+ struct dw_hdmi *hdmi = hdcp->hdmi; -+ u8 val = hdcp->read(hdmi, reg) & ~mask; -+ -+ val |= data & mask; -+ hdcp->write(hdmi, val, reg); -+} -+ -+static void sha_reset(struct sha_t *sha) -+{ -+ u32 i = 0; -+ -+ sha->mindex = 0; -+ sha->mcomputed = false; -+ sha->mcorrupted = false; -+ for (i = 0; i < sizeof(sha->mlength); i++) -+ sha->mlength[i] = 0; -+ -+ sha1_init(sha->mdigest); -+} -+ -+static void sha_processblock(struct sha_t *sha) -+{ -+ u32 array[SHA1_WORKSPACE_WORDS]; -+ -+ sha1_transform(sha->mdigest, sha->mblock, array); -+ sha->mindex = 0; -+} -+ -+static void sha_padmessage(struct sha_t *sha) -+{ -+ /* -+ * Check to see if the current message block is too small to hold -+ * the initial padding bits and length. If so, we will pad the -+ * block, process it, and then continue padding into a second -+ * block. -+ */ -+ if (sha->mindex > 55) { -+ sha->mblock[sha->mindex++] = 0x80; -+ while (sha->mindex < 64) -+ sha->mblock[sha->mindex++] = 0; -+ -+ sha_processblock(sha); -+ while (sha->mindex < 56) -+ sha->mblock[sha->mindex++] = 0; -+ } else { -+ sha->mblock[sha->mindex++] = 0x80; -+ while (sha->mindex < 56) -+ sha->mblock[sha->mindex++] = 0; -+ } -+ -+ /* Store the message length as the last 8 octets */ -+ sha->mblock[56] = sha->mlength[7]; -+ sha->mblock[57] = sha->mlength[6]; -+ sha->mblock[58] = sha->mlength[5]; -+ sha->mblock[59] = sha->mlength[4]; -+ sha->mblock[60] = sha->mlength[3]; -+ sha->mblock[61] = sha->mlength[2]; -+ sha->mblock[62] = sha->mlength[1]; -+ sha->mblock[63] = sha->mlength[0]; -+ -+ sha_processblock(sha); -+} -+ -+static int sha_result(struct sha_t *sha) -+{ -+ if (sha->mcorrupted) -+ return false; -+ -+ if (sha->mcomputed == 0) { -+ sha_padmessage(sha); -+ sha->mcomputed = true; -+ } -+ return true; -+} -+ -+static void sha_input(struct sha_t *sha, const u8 *data, u32 size) -+{ -+ int i = 0; -+ unsigned int j = 0; -+ int rc = true; -+ -+ if (data == 0 || size == 0) -+ return; -+ -+ if (sha->mcomputed || sha->mcorrupted) { -+ sha->mcorrupted = true; -+ return; -+ } -+ while (size-- && !sha->mcorrupted) { -+ sha->mblock[sha->mindex++] = *data; -+ -+ for (i = 0; i < 8; i++) { -+ rc = true; -+ for (j = 0; j < sizeof(sha->mlength); j++) { -+ sha->mlength[j]++; -+ if (sha->mlength[j] != 0) { -+ rc = false; -+ break; -+ } -+ } -+ sha->mcorrupted = (sha->mcorrupted || -+ rc) ? true : false; -+ } -+ /* if corrupted then message is too long */ -+ if (sha->mindex == 64) -+ sha_processblock(sha); -+ data++; -+ } -+} -+ -+static int hdcp_verify_ksv(const u8 *data, u32 size) -+{ -+ u32 i = 0; -+ struct sha_t sha; -+ -+ if ((!data) || (size < (HEADER + SHAMAX))) -+ return false; -+ -+ sha_reset(&sha); -+ sha_input(&sha, data, size - SHAMAX); -+ if (sha_result(&sha) == false) -+ return false; -+ -+ for (i = 0; i < SHAMAX; i++) { -+ if (data[size - SHAMAX + i] != (u8)(sha.mdigest[i / 4] -+ >> ((i % 4) * 8))) -+ return false; -+ } -+ return true; -+} -+ -+static int hdcp_load_keys_cb(struct dw_hdcp *hdcp) -+{ -+ u32 size; -+ u8 hdcp_vendor_data[320]; -+ -+ hdcp->keys = kmalloc(HDCP_KEY_SIZE, GFP_KERNEL); -+ if (!hdcp->keys) -+ return -ENOMEM; -+ -+ hdcp->seeds = kmalloc(HDCP_KEY_SEED_SIZE, GFP_KERNEL); -+ if (!hdcp->seeds) { -+ kfree(hdcp->keys); -+ return -ENOMEM; -+ } -+ -+ size = rk_vendor_read(HDMI_HDCP1X_ID, hdcp_vendor_data, 314); -+ if (size < (HDCP_KEY_SIZE + HDCP_KEY_SEED_SIZE)) { -+ dev_dbg(hdcp->dev, "HDCP: read size %d\n", size); -+ memset(hdcp->keys, 0, HDCP_KEY_SIZE); -+ memset(hdcp->seeds, 0, HDCP_KEY_SEED_SIZE); -+ } else { -+ memcpy(hdcp->keys, hdcp_vendor_data, HDCP_KEY_SIZE); -+ memcpy(hdcp->seeds, hdcp_vendor_data + HDCP_KEY_SIZE, -+ HDCP_KEY_SEED_SIZE); -+ } -+ return 0; -+} -+ -+static int dw_hdmi_hdcp_load_key(struct dw_hdcp *hdcp) -+{ -+ int i, j; -+ int ret, val; -+ void __iomem *reg_rmsts_addr; -+ struct hdcp_keys *hdcp_keys; -+ struct dw_hdmi *hdmi = hdcp->hdmi; -+ -+ if (!hdcp->keys) { -+ ret = hdcp_load_keys_cb(hdcp); -+ if (ret) -+ return ret; -+ } -+ hdcp_keys = hdcp->keys; -+ -+ if (hdcp->reg_io_width == 4) -+ reg_rmsts_addr = hdcp->regs + (HDMI_HDCPREG_RMSTS << 2); -+ else if (hdcp->reg_io_width == 1) -+ reg_rmsts_addr = hdcp->regs + HDMI_HDCPREG_RMSTS; -+ else -+ return -EPERM; -+ -+ /* Disable decryption logic */ -+ hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL); -+ ret = readx_poll_timeout(readl, reg_rmsts_addr, val, -+ val & DPK_WR_OK_STS, 1000, -+ DPK_WR_OK_TIMEOUT_US); -+ if (ret) -+ return ret; -+ hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK6); -+ hdcp->write(hdmi, 0, HDMI_HDCPREG_DPK5); -+ -+ /* The useful data in ksv should be 5 byte */ -+ for (i = 4; i >= 0; i--) -+ hdcp->write(hdmi, hdcp_keys->KSV[i], HDMI_HDCPREG_DPK0 + i); -+ ret = readx_poll_timeout(readl, reg_rmsts_addr, val, -+ val & DPK_WR_OK_STS, 1000, -+ DPK_WR_OK_TIMEOUT_US); -+ -+ if (ret) -+ return ret; -+ -+ /* Enable decryption logic */ -+ if (hdcp->seeds) { -+ hdcp->write(hdmi, 1, HDMI_HDCPREG_RMCTL); -+ hdcp->write(hdmi, hdcp->seeds[0], HDMI_HDCPREG_SEED1); -+ hdcp->write(hdmi, hdcp->seeds[1], HDMI_HDCPREG_SEED0); -+ } else { -+ hdcp->write(hdmi, 0, HDMI_HDCPREG_RMCTL); -+ } -+ -+ /* Write encrypt device private key */ -+ for (i = 0; i < DW_HDMI_HDCP_DPK_LEN - 6; i += 7) { -+ for (j = 6; j >= 0; j--) -+ hdcp->write(hdmi, hdcp_keys->devicekey[i + j], -+ HDMI_HDCPREG_DPK0 + j); -+ ret = readx_poll_timeout(readl, reg_rmsts_addr, val, -+ val & DPK_WR_OK_STS, 1000, -+ DPK_WR_OK_TIMEOUT_US); -+ -+ if (ret) -+ return ret; -+ } -+ return 0; -+} -+ -+static int dw_hdmi_hdcp_start(struct dw_hdcp *hdcp) -+{ -+ struct dw_hdmi *hdmi = hdcp->hdmi; -+ -+ if (!hdcp->enable) -+ return -EPERM; -+ -+ if (!(hdcp->read(hdmi, HDMI_HDCPREG_RMSTS) & 0x3f)) -+ dw_hdmi_hdcp_load_key(hdcp); -+ -+ hdcp_modb(hdcp, HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE, -+ HDMI_FC_INVIDCONF_HDCP_KEEPOUT_MASK, -+ HDMI_FC_INVIDCONF); -+ -+ hdcp->remaining_times = hdcp->retry_times; -+ if (hdcp->read(hdmi, HDMI_CONFIG1_ID) & HDMI_A_HDCP22_MASK) { -+ if (hdcp->hdcp2_enable == 0) { -+ hdcp_modb(hdcp, HDMI_HDCP2_OVR_ENABLE | -+ HDMI_HDCP2_FORCE_DISABLE, -+ HDMI_HDCP2_OVR_EN_MASK | -+ HDMI_HDCP2_FORCE_MASK, -+ HDMI_HDCP2REG_CTRL); -+ hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MASK); -+ hdcp->write(hdmi, 0xff, HDMI_HDCP2REG_MUTE); -+ } else { -+ hdcp_modb(hdcp, HDMI_HDCP2_OVR_DISABLE | -+ HDMI_HDCP2_FORCE_DISABLE, -+ HDMI_HDCP2_OVR_EN_MASK | -+ HDMI_HDCP2_FORCE_MASK, -+ HDMI_HDCP2REG_CTRL); -+ hdcp->write(hdmi, 0x00, HDMI_HDCP2REG_MASK); -+ hdcp->write(hdmi, 0x00, HDMI_HDCP2REG_MUTE); -+ } -+ } -+ -+ hdcp->write(hdmi, 0x40, HDMI_A_OESSWCFG); -+ hdcp_modb(hdcp, HDMI_A_HDCPCFG0_BYPENCRYPTION_DISABLE | -+ HDMI_A_HDCPCFG0_EN11FEATURE_DISABLE | -+ HDMI_A_HDCPCFG0_SYNCRICHECK_ENABLE, -+ HDMI_A_HDCPCFG0_BYPENCRYPTION_MASK | -+ HDMI_A_HDCPCFG0_EN11FEATURE_MASK | -+ HDMI_A_HDCPCFG0_SYNCRICHECK_MASK, HDMI_A_HDCPCFG0); -+ -+ hdcp_modb(hdcp, HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_ENABLE | -+ HDMI_A_HDCPCFG1_PH2UPSHFTENC_ENABLE, -+ HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK | -+ HDMI_A_HDCPCFG1_PH2UPSHFTENC_MASK, HDMI_A_HDCPCFG1); -+ -+ /* Reset HDCP Engine */ -+ if (hdcp->read(hdmi, HDMI_MC_CLKDIS) & HDMI_MC_CLKDIS_HDCPCLK_MASK) { -+ hdcp_modb(hdcp, HDMI_A_HDCPCFG1_SWRESET_ASSERT, -+ HDMI_A_HDCPCFG1_SWRESET_MASK, HDMI_A_HDCPCFG1); -+ } -+ -+ hdcp->write(hdmi, 0x00, HDMI_A_APIINTMSK); -+ hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_ENABLE, -+ HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0); -+ -+ /* -+ * XXX: to sleep 100ms here between output hdmi and enable hdcpclk, -+ * otherwise hdcp auth fail when Connect to repeater -+ */ -+ msleep(100); -+ hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_ENABLE, -+ HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS); -+ -+ hdcp->status = DW_HDCP_AUTH_START; -+ dev_dbg(hdcp->dev, "%s success\n", __func__); -+ return 0; -+} -+ -+static int dw_hdmi_hdcp_stop(struct dw_hdcp *hdcp) -+{ -+ struct dw_hdmi *hdmi = hdcp->hdmi; -+ -+ if (!hdcp->enable) -+ return -EPERM; -+ -+ hdcp_modb(hdcp, HDMI_MC_CLKDIS_HDCPCLK_DISABLE, -+ HDMI_MC_CLKDIS_HDCPCLK_MASK, HDMI_MC_CLKDIS); -+ hdcp->write(hdmi, 0xff, HDMI_A_APIINTMSK); -+ -+ hdcp_modb(hdcp, HDMI_A_HDCPCFG0_RXDETECT_DISABLE, -+ HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0); -+ -+ hdcp_modb(hdcp, HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE | -+ HDMI_A_SRMCTRL_KSV_UPDATE_DISABLE, -+ HDMI_A_SRMCTRL_SHA1_FAIL_MASK | -+ HDMI_A_SRMCTRL_KSV_UPDATE_MASK, HDMI_A_SRMCTRL); -+ -+ hdcp->status = DW_HDCP_DISABLED; -+ return 0; -+} -+ -+static int dw_hdmi_hdcp_ksvsha1(struct dw_hdcp *hdcp) -+{ -+ int rc = 0, value, list, i; -+ char bstaus0, bstaus1; -+ char *ksvlistbuf; -+ struct dw_hdmi *hdmi = hdcp->hdmi; -+ -+ hdcp_modb(hdcp, HDMI_A_SRMCTRL_KSV_MEM_REQ_ENABLE, -+ HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK, HDMI_A_SRMCTRL); -+ -+ list = 20; -+ do { -+ value = hdcp->read(hdmi, HDMI_A_SRMCTRL); -+ usleep_range(500, 1000); -+ } while ((value & HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK) == 0 && --list); -+ -+ if ((value & HDMI_A_SRMCTRL_KSV_MEM_ACCESS_MASK) == 0) { -+ dev_err(hdcp->dev, "KSV memory can not access\n"); -+ rc = -EPERM; -+ goto out; -+ } -+ -+ hdcp->read(hdmi, HDMI_A_SRM_BASE); -+ bstaus0 = hdcp->read(hdmi, HDMI_A_SRM_BASE + 1); -+ bstaus1 = hdcp->read(hdmi, HDMI_A_SRM_BASE + 2); -+ -+ if (bstaus0 & HDMI_A_SRM_BASE_MAX_DEVS_EXCEEDED) { -+ dev_err(hdcp->dev, "MAX_DEVS_EXCEEDED\n"); -+ rc = -EPERM; -+ goto out; -+ } -+ -+ list = bstaus0 & HDMI_A_SRM_BASE_DEVICE_COUNT; -+ if (list > MAX_DOWNSTREAM_DEVICE_NUM) { -+ dev_err(hdcp->dev, "MAX_DOWNSTREAM_DEVICE_NUM\n"); -+ rc = -EPERM; -+ goto out; -+ } -+ if (bstaus1 & HDMI_A_SRM_BASE_MAX_CASCADE_EXCEEDED) { -+ dev_err(hdcp->dev, "MAX_CASCADE_EXCEEDED\n"); -+ rc = -EPERM; -+ goto out; -+ } -+ -+ value = (list * KSV_LEN) + HEADER + SHAMAX; -+ ksvlistbuf = kmalloc(value, GFP_KERNEL); -+ if (!ksvlistbuf) { -+ rc = -ENOMEM; -+ goto out; -+ } -+ -+ ksvlistbuf[(list * KSV_LEN)] = bstaus0; -+ ksvlistbuf[(list * KSV_LEN) + 1] = bstaus1; -+ for (i = 2; i < value; i++) { -+ if (i < HEADER) /* BSTATUS & M0 */ -+ ksvlistbuf[(list * KSV_LEN) + i] = -+ hdcp->read(hdmi, HDMI_A_SRM_BASE + i + 1); -+ else if (i < (HEADER + (list * KSV_LEN))) /* KSV list */ -+ ksvlistbuf[i - HEADER] = -+ hdcp->read(hdmi, HDMI_A_SRM_BASE + i + 1); -+ else /* SHA */ -+ ksvlistbuf[i] = -+ hdcp->read(hdmi, HDMI_A_SRM_BASE + i + 1); -+ } -+ if (hdcp_verify_ksv(ksvlistbuf, value) == true) { -+ rc = 0; -+ dev_dbg(hdcp->dev, "ksv check valid\n"); -+ } else { -+ dev_err(hdcp->dev, "ksv check invalid\n"); -+ rc = -1; -+ } -+ kfree(ksvlistbuf); -+out: -+ hdcp_modb(hdcp, HDMI_A_SRMCTRL_KSV_MEM_REQ_DISABLE, -+ HDMI_A_SRMCTRL_KSV_MEM_REQ_MASK, HDMI_A_SRMCTRL); -+ return rc; -+} -+ -+static void dw_hdmi_hdcp_2nd_auth(struct dw_hdcp *hdcp) -+{ -+ if (dw_hdmi_hdcp_ksvsha1(hdcp)) -+ hdcp_modb(hdcp, HDMI_A_SRMCTRL_SHA1_FAIL_ENABLE | -+ HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE, -+ HDMI_A_SRMCTRL_SHA1_FAIL_MASK | -+ HDMI_A_SRMCTRL_KSV_UPDATE_MASK, HDMI_A_SRMCTRL); -+ else -+ hdcp_modb(hdcp, HDMI_A_SRMCTRL_SHA1_FAIL_DISABLE | -+ HDMI_A_SRMCTRL_KSV_UPDATE_ENABLE, -+ HDMI_A_SRMCTRL_SHA1_FAIL_MASK | -+ HDMI_A_SRMCTRL_KSV_UPDATE_MASK, HDMI_A_SRMCTRL); -+} -+ -+static void dw_hdmi_hdcp_isr(struct dw_hdcp *hdcp, int hdcp_int) -+{ -+ dev_dbg(hdcp->dev, "hdcp_int is 0x%02x\n", hdcp_int); -+ if (hdcp_int & HDMI_A_APIINTSTAT_KSVSHA1_CALC_INT) { -+ dev_dbg(hdcp->dev, "hdcp sink is a repeater\n"); -+ dw_hdmi_hdcp_2nd_auth(hdcp); -+ } -+ if (hdcp_int & 0x40) { -+ hdcp->status = DW_HDCP_AUTH_FAIL; -+ if (hdcp->remaining_times > 1) -+ hdcp->remaining_times--; -+ else if (hdcp->remaining_times == 1) -+ hdcp_modb(hdcp, -+ HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE, -+ HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, -+ HDMI_A_HDCPCFG1); -+ } -+ if (hdcp_int & 0x80) { -+ dev_dbg(hdcp->dev, "hdcp auth success\n"); -+ hdcp->status = DW_HDCP_AUTH_SUCCESS; -+ } -+} -+ -+static ssize_t hdcp_enable_read(struct device *device, -+ struct device_attribute *attr, char *buf) -+{ -+ bool enable = 0; -+ struct dw_hdcp *hdcp = g_hdcp; -+ -+ if (hdcp) -+ enable = hdcp->enable; -+ -+ return snprintf(buf, PAGE_SIZE, "%d\n", enable); -+} -+ -+static ssize_t hdcp_enable_write(struct device *device, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ bool enable; -+ struct dw_hdcp *hdcp = g_hdcp; -+ -+ if (!hdcp) -+ return -EINVAL; -+ -+ if (kstrtobool(buf, &enable)) -+ return -EINVAL; -+ -+ if (hdcp->enable != enable) { -+ if (enable) { -+ hdcp->enable = enable; -+ if (hdcp->read(hdcp->hdmi, HDMI_PHY_STAT0) & -+ HDMI_PHY_HPD) -+ dw_hdmi_hdcp_start(hdcp); -+ } else { -+ dw_hdmi_hdcp_stop(hdcp); -+ hdcp->enable = enable; -+ } -+ } -+ -+ return count; -+} -+ -+static DEVICE_ATTR(enable, 0644, hdcp_enable_read, hdcp_enable_write); -+ -+static ssize_t hdcp_trytimes_read(struct device *device, -+ struct device_attribute *attr, char *buf) -+{ -+ int trytimes = 0; -+ struct dw_hdcp *hdcp = g_hdcp; -+ -+ if (hdcp) -+ trytimes = hdcp->retry_times; -+ -+ return snprintf(buf, PAGE_SIZE, "%d\n", trytimes); -+} -+ -+static ssize_t hdcp_trytimes_write(struct device *device, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ int trytimes; -+ struct dw_hdcp *hdcp = g_hdcp; -+ -+ if (!hdcp) -+ return -EINVAL; -+ -+ if (kstrtoint(buf, 0, &trytimes)) -+ return -EINVAL; -+ -+ if (hdcp->retry_times != trytimes) { -+ hdcp->retry_times = trytimes; -+ hdcp->remaining_times = hdcp->retry_times; -+ } -+ -+ return count; -+} -+ -+static DEVICE_ATTR(trytimes, 0644, hdcp_trytimes_read, hdcp_trytimes_write); -+ -+static ssize_t hdcp_status_read(struct device *device, -+ struct device_attribute *attr, char *buf) -+{ -+ int status = DW_HDCP_DISABLED; -+ struct dw_hdcp *hdcp = g_hdcp; -+ -+ if (hdcp) -+ status = hdcp->status; -+ -+ if (status == DW_HDCP_DISABLED) -+ return snprintf(buf, PAGE_SIZE, "hdcp disable\n"); -+ else if (status == DW_HDCP_AUTH_START) -+ return snprintf(buf, PAGE_SIZE, "hdcp_auth_start\n"); -+ else if (status == DW_HDCP_AUTH_SUCCESS) -+ return snprintf(buf, PAGE_SIZE, "hdcp_auth_success\n"); -+ else if (status == DW_HDCP_AUTH_FAIL) -+ return snprintf(buf, PAGE_SIZE, "hdcp_auth_fail\n"); -+ else -+ return snprintf(buf, PAGE_SIZE, "unknown status\n"); -+} -+ -+static DEVICE_ATTR(status, 0444, hdcp_status_read, NULL); -+ -+static int dw_hdmi_hdcp_probe(struct platform_device *pdev) -+{ -+ int ret = 0; -+ struct dw_hdcp *hdcp = pdev->dev.platform_data; -+ -+ g_hdcp = hdcp; -+ hdcp->mdev.minor = MISC_DYNAMIC_MINOR; -+ hdcp->mdev.name = "hdmi_hdcp1x"; -+ hdcp->mdev.mode = 0666; -+ -+ if (misc_register(&hdcp->mdev)) { -+ dev_err(&pdev->dev, "HDCP: Could not add character driver\n"); -+ return -EINVAL; -+ } -+ -+ ret = device_create_file(hdcp->mdev.this_device, &dev_attr_enable); -+ if (ret) { -+ dev_err(&pdev->dev, "HDCP: Could not add sys file enable\n"); -+ ret = -EINVAL; -+ goto error0; -+ } -+ -+ ret = device_create_file(hdcp->mdev.this_device, &dev_attr_trytimes); -+ if (ret) { -+ dev_err(&pdev->dev, "HDCP: Could not add sys file trytimes\n"); -+ ret = -EINVAL; -+ goto error1; -+ } -+ -+ ret = device_create_file(hdcp->mdev.this_device, &dev_attr_status); -+ if (ret) { -+ dev_err(&pdev->dev, "HDCP: Could not add sys file status\n"); -+ ret = -EINVAL; -+ goto error2; -+ } -+ -+ /* retry time if hdcp auth fail. unlimited time if set 0 */ -+ hdcp->retry_times = 0; -+ hdcp->dev = &pdev->dev; -+ hdcp->hdcp_start = dw_hdmi_hdcp_start; -+ hdcp->hdcp_stop = dw_hdmi_hdcp_stop; -+ hdcp->hdcp_isr = dw_hdmi_hdcp_isr; -+ dev_dbg(hdcp->dev, "%s success\n", __func__); -+ return 0; -+ -+error2: -+ device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes); -+error1: -+ device_remove_file(hdcp->mdev.this_device, &dev_attr_enable); -+error0: -+ misc_deregister(&hdcp->mdev); -+ return ret; -+} -+ -+static int dw_hdmi_hdcp_remove(struct platform_device *pdev) -+{ -+ struct dw_hdcp *hdcp = pdev->dev.platform_data; -+ -+ device_remove_file(hdcp->mdev.this_device, &dev_attr_trytimes); -+ device_remove_file(hdcp->mdev.this_device, &dev_attr_enable); -+ device_remove_file(hdcp->mdev.this_device, &dev_attr_status); -+ misc_deregister(&hdcp->mdev); -+ -+ kfree(hdcp->keys); -+ kfree(hdcp->seeds); -+ -+ return 0; -+} -+ -+static struct platform_driver dw_hdmi_hdcp_driver = { -+ .probe = dw_hdmi_hdcp_probe, -+ .remove = dw_hdmi_hdcp_remove, -+ .driver = { -+ .name = DW_HDCP_DRIVER_NAME, -+ }, -+}; -+ -+module_platform_driver(dw_hdmi_hdcp_driver); -+MODULE_DESCRIPTION("DW HDMI transmitter HDCP driver"); -+MODULE_LICENSE("GPL"); -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h -new file mode 100755 -index 000000000..d138f91f3 ---- /dev/null -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-hdcp.h -@@ -0,0 +1,54 @@ -+/* -+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd -+ * Author Huicong Xu -+ * -+ * This software is licensed under the terms of the GNU General Public -+ * License version 2, as published by the Free Software Foundation, and -+ * may be copied, distributed, and modified under those terms. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ */ -+ -+#ifndef DW_HDMI_HDCP_H -+#define DW_HDMI_HDCP_H -+ -+#include -+ -+#define DW_HDCP_DRIVER_NAME "dw-hdmi-hdcp" -+#define HDCP_PRIVATE_KEY_SIZE 280 -+#define HDCP_KEY_SHA_SIZE 20 -+ -+struct hdcp_keys { -+ u8 KSV[8]; -+ u8 devicekey[HDCP_PRIVATE_KEY_SIZE]; -+ u8 sha1[HDCP_KEY_SHA_SIZE]; -+}; -+ -+struct dw_hdcp { -+ bool enable; -+ int retry_times; -+ int remaining_times; -+ char *seeds; -+ int invalidkey; -+ char *invalidkeys; -+ int hdcp2_enable; -+ int status; -+ u32 reg_io_width; -+ -+ struct miscdevice mdev; -+ struct hdcp_keys *keys; -+ struct device *dev; -+ struct dw_hdmi *hdmi; -+ void __iomem *regs; -+ -+ void (*write)(struct dw_hdmi *hdmi, u8 val, int offset); -+ u8 (*read)(struct dw_hdmi *hdmi, int offset); -+ int (*hdcp_start)(struct dw_hdcp *hdcp); -+ int (*hdcp_stop)(struct dw_hdcp *hdcp); -+ void (*hdcp_isr)(struct dw_hdcp *hdcp, int hdcp_int); -+}; -+ -+#endif -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h -new file mode 100755 -index 000000000..93f1a4295 ---- /dev/null -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-audio.h -@@ -0,0 +1,29 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (c) 2021 Rockchip Electronics Co. Ltd. -+ * Author: Sugar Zhang -+ */ -+ -+#ifndef DW_HDMI_QP_AUDIO_H -+#define DW_HDMI_QP_AUDIO_H -+ -+struct dw_hdmi_qp; -+ -+struct dw_hdmi_qp_audio_data { -+ phys_addr_t phys; -+ void __iomem *base; -+ int irq; -+ struct dw_hdmi_qp *hdmi; -+ u8 *eld; -+}; -+ -+struct dw_hdmi_qp_i2s_audio_data { -+ struct dw_hdmi_qp *hdmi; -+ u8 *eld; -+ -+ void (*write)(struct dw_hdmi_qp *hdmi, u32 val, int offset); -+ u32 (*read)(struct dw_hdmi_qp *hdmi, int offset); -+ void (*mod)(struct dw_hdmi_qp *hdmi, u32 val, u32 mask, u32 reg); -+}; -+ -+#endif -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c -new file mode 100755 -index 000000000..8aa581555 ---- /dev/null -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp-i2s-audio.c -@@ -0,0 +1,262 @@ -+// SPDX-License-Identifier: GPL-2.0 -+/* -+ * dw-hdmi-qp-i2s-audio.c -+ * -+ * Copyright (c) 2021 Rockchip Electronics Co. Ltd. -+ * Author: Sugar Zhang -+ */ -+ -+#include -+#include -+ -+#include -+#include -+ -+#include -+ -+#include "dw-hdmi-qp.h" -+#include "dw-hdmi-qp-audio.h" -+ -+#define DRIVER_NAME "dw-hdmi-qp-i2s-audio" -+ -+static inline void hdmi_write(struct dw_hdmi_qp_i2s_audio_data *audio, -+ u32 val, int offset) -+{ -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ -+ audio->write(hdmi, val, offset); -+} -+ -+static inline u32 hdmi_read(struct dw_hdmi_qp_i2s_audio_data *audio, int offset) -+{ -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ -+ return audio->read(hdmi, offset); -+} -+ -+static inline void hdmi_mod(struct dw_hdmi_qp_i2s_audio_data *audio, -+ u32 data, u32 mask, u32 reg) -+{ -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ -+ return audio->mod(hdmi, data, mask, reg); -+} -+ -+static inline bool is_dw_hdmi_qp_clk_off(struct dw_hdmi_qp_i2s_audio_data *audio) -+{ -+ u32 sta = hdmi_read(audio, CMU_STATUS); -+ -+ return (sta & (AUDCLK_OFF | LINKQPCLK_OFF | VIDQPCLK_OFF)); -+} -+ -+static int dw_hdmi_qp_i2s_hw_params(struct device *dev, void *data, -+ struct hdmi_codec_daifmt *fmt, -+ struct hdmi_codec_params *hparms) -+{ -+ struct dw_hdmi_qp_i2s_audio_data *audio = data; -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ u32 conf0 = 0; -+ -+ if (is_dw_hdmi_qp_clk_off(audio)) -+ return 0; -+ -+ if (fmt->bit_clk_master | fmt->frame_clk_master) { -+ dev_err(dev, "unsupported clock settings\n"); -+ return -EINVAL; -+ } -+ -+ /* Reset the audio data path of the AVP */ -+ hdmi_write(audio, AVP_DATAPATH_PACKET_AUDIO_SWINIT_P, GLOBAL_SWRESET_REQUEST); -+ -+ /* Clear the audio FIFO */ -+ hdmi_write(audio, AUDIO_FIFO_CLR_P, AUDIO_INTERFACE_CONTROL0); -+ -+ /* Disable AUDS, ACR, AUDI, AMD */ -+ hdmi_mod(audio, 0, -+ PKTSCHED_ACR_TX_EN | PKTSCHED_AUDS_TX_EN | -+ PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN, -+ PKTSCHED_PKT_EN); -+ -+ /* Select I2S interface as the audio source */ -+ hdmi_mod(audio, AUD_IF_I2S, AUD_IF_SEL_MSK, AUDIO_INTERFACE_CONFIG0); -+ -+ /* Enable the active i2s lanes */ -+ switch (hparms->channels) { -+ case 7 ... 8: -+ conf0 |= I2S_LINES_EN(3); -+ fallthrough; -+ case 5 ... 6: -+ conf0 |= I2S_LINES_EN(2); -+ fallthrough; -+ case 3 ... 4: -+ conf0 |= I2S_LINES_EN(1); -+ fallthrough; -+ default: -+ conf0 |= I2S_LINES_EN(0); -+ break; -+ } -+ -+ hdmi_mod(audio, conf0, I2S_LINES_EN_MSK, AUDIO_INTERFACE_CONFIG0); -+ -+ /* -+ * Enable bpcuv generated internally for L-PCM, or received -+ * from stream for NLPCM/HBR. -+ */ -+ switch (fmt->bit_fmt) { -+ case SNDRV_PCM_FORMAT_IEC958_SUBFRAME_LE: -+ conf0 = (hparms->channels == 8) ? AUD_HBR : AUD_ASP; -+ conf0 |= I2S_BPCUV_RCV_EN; -+ break; -+ default: -+ conf0 = AUD_ASP | I2S_BPCUV_RCV_DIS; -+ break; -+ } -+ -+ hdmi_mod(audio, conf0, I2S_BPCUV_RCV_MSK | AUD_FORMAT_MSK, -+ AUDIO_INTERFACE_CONFIG0); -+ -+ /* Enable audio FIFO auto clear when overflow */ -+ hdmi_mod(audio, AUD_FIFO_INIT_ON_OVF_EN, AUD_FIFO_INIT_ON_OVF_MSK, -+ AUDIO_INTERFACE_CONFIG0); -+ -+ dw_hdmi_qp_set_sample_rate(hdmi, hparms->sample_rate); -+ dw_hdmi_qp_set_channel_status(hdmi, hparms->iec.status); -+ dw_hdmi_qp_set_channel_count(hdmi, hparms->channels); -+ dw_hdmi_qp_set_channel_allocation(hdmi, hparms->cea.channel_allocation); -+ -+ /* Enable ACR, AUDI, AMD */ -+ hdmi_mod(audio, -+ PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN, -+ PKTSCHED_ACR_TX_EN | PKTSCHED_AUDI_TX_EN | PKTSCHED_AMD_TX_EN, -+ PKTSCHED_PKT_EN); -+ -+ /* Enable AUDS */ -+ hdmi_mod(audio, PKTSCHED_AUDS_TX_EN, PKTSCHED_AUDS_TX_EN, PKTSCHED_PKT_EN); -+ -+ return 0; -+} -+ -+static int dw_hdmi_qp_i2s_audio_startup(struct device *dev, void *data) -+{ -+ struct dw_hdmi_qp_i2s_audio_data *audio = data; -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ -+ if (is_dw_hdmi_qp_clk_off(audio)) -+ return 0; -+ -+ dw_hdmi_qp_audio_enable(hdmi); -+ -+ return 0; -+} -+ -+static void dw_hdmi_qp_i2s_audio_shutdown(struct device *dev, void *data) -+{ -+ struct dw_hdmi_qp_i2s_audio_data *audio = data; -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ -+ if (is_dw_hdmi_qp_clk_off(audio)) -+ return; -+ -+ dw_hdmi_qp_audio_disable(hdmi); -+} -+ -+static int dw_hdmi_qp_i2s_get_eld(struct device *dev, void *data, uint8_t *buf, -+ size_t len) -+{ -+ struct dw_hdmi_qp_i2s_audio_data *audio = data; -+ -+ memcpy(buf, audio->eld, min_t(size_t, MAX_ELD_BYTES, len)); -+ -+ return 0; -+} -+ -+static int dw_hdmi_qp_i2s_get_dai_id(struct snd_soc_component *component, -+ struct device_node *endpoint) -+{ -+ struct of_endpoint of_ep; -+ int ret; -+ -+ ret = of_graph_parse_endpoint(endpoint, &of_ep); -+ if (ret < 0) -+ return ret; -+ -+ /* -+ * HDMI sound should be located as reg = <2> -+ * Then, it is sound port 0 -+ */ -+ if (of_ep.port == 2) -+ return 0; -+ -+ return -EINVAL; -+} -+ -+static int dw_hdmi_qp_i2s_hook_plugged_cb(struct device *dev, void *data, -+ hdmi_codec_plugged_cb fn, -+ struct device *codec_dev) -+{ -+ struct dw_hdmi_qp_i2s_audio_data *audio = data; -+ struct dw_hdmi_qp *hdmi = audio->hdmi; -+ -+ return dw_hdmi_qp_set_plugged_cb(hdmi, fn, codec_dev); -+} -+ -+static struct hdmi_codec_ops dw_hdmi_qp_i2s_ops = { -+ .hw_params = dw_hdmi_qp_i2s_hw_params, -+ .audio_startup = dw_hdmi_qp_i2s_audio_startup, -+ .audio_shutdown = dw_hdmi_qp_i2s_audio_shutdown, -+ .get_eld = dw_hdmi_qp_i2s_get_eld, -+ .get_dai_id = dw_hdmi_qp_i2s_get_dai_id, -+ .hook_plugged_cb = dw_hdmi_qp_i2s_hook_plugged_cb, -+}; -+ -+static int snd_dw_hdmi_qp_probe(struct platform_device *pdev) -+{ -+ struct dw_hdmi_qp_i2s_audio_data *audio = pdev->dev.platform_data; -+ struct platform_device_info pdevinfo; -+ struct hdmi_codec_pdata pdata; -+ struct platform_device *platform; -+ -+ pdata.ops = &dw_hdmi_qp_i2s_ops; -+ pdata.i2s = 1; -+ pdata.max_i2s_channels = 8; -+ pdata.data = audio; -+ -+ memset(&pdevinfo, 0, sizeof(pdevinfo)); -+ pdevinfo.parent = pdev->dev.parent; -+ pdevinfo.id = PLATFORM_DEVID_AUTO; -+ pdevinfo.name = HDMI_CODEC_DRV_NAME; -+ pdevinfo.data = &pdata; -+ pdevinfo.size_data = sizeof(pdata); -+ pdevinfo.dma_mask = DMA_BIT_MASK(32); -+ -+ platform = platform_device_register_full(&pdevinfo); -+ if (IS_ERR(platform)) -+ return PTR_ERR(platform); -+ -+ dev_set_drvdata(&pdev->dev, platform); -+ -+ return 0; -+} -+ -+static int snd_dw_hdmi_qp_remove(struct platform_device *pdev) -+{ -+ struct platform_device *platform = dev_get_drvdata(&pdev->dev); -+ -+ platform_device_unregister(platform); -+ -+ return 0; -+} -+ -+static struct platform_driver snd_dw_hdmi_qp_driver = { -+ .probe = snd_dw_hdmi_qp_probe, -+ .remove = snd_dw_hdmi_qp_remove, -+ .driver = { -+ .name = DRIVER_NAME, -+ }, -+}; -+module_platform_driver(snd_dw_hdmi_qp_driver); -+ -+MODULE_AUTHOR("Sugar Zhang "); -+MODULE_DESCRIPTION("Synopsis Designware HDMI QP I2S ALSA SoC interface"); -+MODULE_LICENSE("GPL v2"); -+MODULE_ALIAS("platform:" DRIVER_NAME); -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c -new file mode 100755 -index 000000000..cd8150b33 ---- /dev/null -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c -@@ -0,0 +1,2455 @@ -+// SPDX-License-Identifier: GPL-2.0+ -+/* -+ * Copyright (C) Rockchip Electronics Co.Ltd -+ * Author: -+ * Algea Cao -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+ -+#include "dw-hdmi-qp-audio.h" -+#include "dw-hdmi-qp.h" -+ -+#include -+ -+#define DDC_CI_ADDR 0x37 -+#define DDC_SEGMENT_ADDR 0x30 -+ -+#define HDMI_EDID_LEN 512 -+ -+/* DW-HDMI Controller >= 0x200a are at least compliant with SCDC version 1 */ -+#define SCDC_MIN_SOURCE_VERSION 0x1 -+ -+#define HDMI14_MAX_TMDSCLK 340000000 -+ -+static const unsigned int dw_hdmi_cable[] = { -+ EXTCON_DISP_HDMI, -+ EXTCON_NONE, -+}; -+ -+/* -+ * Unless otherwise noted, entries in this table are 100% optimization. -+ * Values can be obtained from hdmi_compute_n() but that function is -+ * slow so we pre-compute values we expect to see. -+ * -+ * All 32k and 48k values are expected to be the same (due to the way -+ * the math works) for any rate that's an exact kHz. -+ */ -+static const struct dw_hdmi_audio_tmds_n common_tmds_n_table[] = { -+ { .tmds = 25175000, .n_32k = 4096, .n_44k1 = 12854, .n_48k = 6144, }, -+ { .tmds = 25200000, .n_32k = 4096, .n_44k1 = 5656, .n_48k = 6144, }, -+ { .tmds = 27000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 28320000, .n_32k = 4096, .n_44k1 = 5586, .n_48k = 6144, }, -+ { .tmds = 30240000, .n_32k = 4096, .n_44k1 = 5642, .n_48k = 6144, }, -+ { .tmds = 31500000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, -+ { .tmds = 32000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, -+ { .tmds = 33750000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, -+ { .tmds = 36000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ { .tmds = 40000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, -+ { .tmds = 49500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 50000000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, -+ { .tmds = 54000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ { .tmds = 65000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 68250000, .n_32k = 4096, .n_44k1 = 5376, .n_48k = 6144, }, -+ { .tmds = 71000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 72000000, .n_32k = 4096, .n_44k1 = 5635, .n_48k = 6144, }, -+ { .tmds = 73250000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, -+ { .tmds = 74250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, -+ { .tmds = 75000000, .n_32k = 4096, .n_44k1 = 5880, .n_48k = 6144, }, -+ { .tmds = 78750000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, -+ { .tmds = 78800000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, -+ { .tmds = 79500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, -+ { .tmds = 83500000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 85500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 88750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, -+ { .tmds = 97750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, -+ { .tmds = 101000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 106500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, -+ { .tmds = 108000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ { .tmds = 115500000, .n_32k = 4096, .n_44k1 = 5712, .n_48k = 6144, }, -+ { .tmds = 119000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, -+ { .tmds = 135000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 146250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, -+ { .tmds = 148500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 154000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, -+ { .tmds = 162000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ -+ /* For 297 MHz+ HDMI spec have some other rule for setting N */ -+ { .tmds = 297000000, .n_32k = 3073, .n_44k1 = 4704, .n_48k = 5120, }, -+ { .tmds = 594000000, .n_32k = 3073, .n_44k1 = 9408, .n_48k = 10240, }, -+ -+ /* End of table */ -+ { .tmds = 0, .n_32k = 0, .n_44k1 = 0, .n_48k = 0, }, -+}; -+ -+static const struct drm_display_mode dw_hdmi_default_modes[] = { -+ /* 16 - 1920x1080@60Hz 16:9 */ -+ { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008, -+ 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 2 - 720x480@60Hz 4:3 */ -+ { DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, -+ 798, 858, 0, 480, 489, 495, 525, 0, -+ DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, -+ /* 4 - 1280x720@60Hz 16:9 */ -+ { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390, -+ 1430, 1650, 0, 720, 725, 730, 750, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 31 - 1920x1080@50Hz 16:9 */ -+ { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448, -+ 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 19 - 1280x720@50Hz 16:9 */ -+ { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720, -+ 1760, 1980, 0, 720, 725, 730, 750, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 17 - 720x576@50Hz 4:3 */ -+ { DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, -+ 796, 864, 0, 576, 581, 586, 625, 0, -+ DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, -+ /* 2 - 720x480@60Hz 4:3 */ -+ { DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, -+ 798, 858, 0, 480, 489, 495, 525, 0, -+ DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, -+}; -+ -+enum frl_mask { -+ FRL_3GBPS_3LANE = 1, -+ FRL_6GBPS_3LANE, -+ FRL_6GBPS_4LANE, -+ FRL_8GBPS_4LANE, -+ FRL_10GBPS_4LANE, -+ FRL_12GBPS_4LANE, -+}; -+ -+struct hdmi_vmode_qp { -+ bool mdataenablepolarity; -+ -+ unsigned int previous_pixelclock; -+ unsigned long mpixelclock; -+ unsigned int mpixelrepetitioninput; -+ unsigned int mpixelrepetitionoutput; -+ unsigned long previous_tmdsclock; -+ unsigned int mtmdsclock; -+}; -+ -+struct hdmi_qp_data_info { -+ unsigned int enc_in_bus_format; -+ unsigned int enc_out_bus_format; -+ unsigned int enc_in_encoding; -+ unsigned int enc_out_encoding; -+ unsigned int quant_range; -+ unsigned int pix_repet_factor; -+ struct hdmi_vmode_qp video_mode; -+ bool update; -+}; -+ -+struct dw_hdmi_qp_i2c { -+ struct i2c_adapter adap; -+ -+ struct mutex lock; /* used to serialize data transfers */ -+ struct completion cmp; -+ u32 stat; -+ -+ u8 slave_reg; -+ bool is_regaddr; -+ bool is_segment; -+ -+ unsigned int scl_high_ns; -+ unsigned int scl_low_ns; -+}; -+ -+struct dw_hdmi_phy_data { -+ enum dw_hdmi_phy_type type; -+ const char *name; -+ unsigned int gen; -+ bool has_svsret; -+ int (*configure)(struct dw_hdmi_qp *hdmi, -+ const struct dw_hdmi_plat_data *pdata, -+ unsigned long mpixelclock); -+}; -+ -+struct dw_hdmi_qp { -+ struct drm_connector connector; -+ struct drm_bridge bridge; -+ struct platform_device *hdcp_dev; -+ struct platform_device *audio; -+ -+ struct device *dev; -+ struct dw_hdmi_qp_i2c *i2c; -+ -+ struct hdmi_qp_data_info hdmi_data; -+ const struct dw_hdmi_plat_data *plat_data; -+ -+ int vic; -+ int main_irq; -+ int avp_irq; -+ int earc_irq; -+ -+ u8 edid[HDMI_EDID_LEN]; -+ -+ struct { -+ const struct dw_hdmi_qp_phy_ops *ops; -+ const char *name; -+ void *data; -+ bool enabled; -+ } phy; -+ -+ struct drm_display_mode previous_mode; -+ -+ struct i2c_adapter *ddc; -+ void __iomem *regs; -+ bool sink_is_hdmi; -+ bool sink_has_audio; -+ bool hpd_state; -+ -+ struct mutex mutex; /* for state below and previous_mode */ -+ struct drm_connector *curr_conn;/* current connector (only valid when !disabled) */ -+ enum drm_connector_force force; /* mutex-protected force state */ -+ bool disabled; /* DRM has disabled our bridge */ -+ bool bridge_is_on; /* indicates the bridge is on */ -+ bool rxsense; /* rxsense state */ -+ u8 phy_mask; /* desired phy int mask settings */ -+ u8 mc_clkdis; /* clock disable register */ -+ -+ u32 scdc_intr; -+ u32 flt_intr; -+ u32 earc_intr; -+ -+ spinlock_t audio_lock; -+ struct mutex audio_mutex; -+ unsigned int sample_rate; -+ unsigned int audio_cts; -+ unsigned int audio_n; -+ bool audio_enable; -+ void (*enable_audio)(struct dw_hdmi_qp *hdmi); -+ void (*disable_audio)(struct dw_hdmi_qp *hdmi); -+ -+ struct dentry *debugfs_dir; -+ bool scramble_low_rates; -+ -+ struct extcon_dev *extcon; -+ -+ struct regmap *regm; -+ -+ bool initialized; /* hdmi is enabled before bind */ -+ struct completion flt_cmp; -+ struct completion earc_cmp; -+ -+ struct cec_notifier *cec_notifier; -+ struct cec_adapter *cec_adap; -+ struct mutex cec_notifier_mutex; -+ -+ hdmi_codec_plugged_cb plugged_cb; -+ struct device *codec_dev; -+ enum drm_connector_status last_connector_result; -+}; -+ -+static inline void hdmi_writel(struct dw_hdmi_qp *hdmi, u32 val, int offset) -+{ -+ regmap_write(hdmi->regm, offset, val); -+} -+ -+static inline u32 hdmi_readl(struct dw_hdmi_qp *hdmi, int offset) -+{ -+ unsigned int val = 0; -+ -+ regmap_read(hdmi->regm, offset, &val); -+ -+ return val; -+} -+ -+static void handle_plugged_change(struct dw_hdmi_qp *hdmi, bool plugged) -+{ -+ if (hdmi->plugged_cb && hdmi->codec_dev) -+ hdmi->plugged_cb(hdmi->codec_dev, plugged); -+} -+ -+int dw_hdmi_qp_set_plugged_cb(struct dw_hdmi_qp *hdmi, hdmi_codec_plugged_cb fn, -+ struct device *codec_dev) -+{ -+ bool plugged; -+ -+ mutex_lock(&hdmi->mutex); -+ hdmi->plugged_cb = fn; -+ hdmi->codec_dev = codec_dev; -+ plugged = hdmi->last_connector_result == connector_status_connected; -+ handle_plugged_change(hdmi, plugged); -+ mutex_unlock(&hdmi->mutex); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_plugged_cb); -+ -+static void hdmi_modb(struct dw_hdmi_qp *hdmi, u32 data, u32 mask, u32 reg) -+{ -+ regmap_update_bits(hdmi->regm, reg, mask, data); -+} -+ -+static void hdmi_set_cts_n(struct dw_hdmi_qp *hdmi, unsigned int cts, -+ unsigned int n) -+{ -+ /* Set N */ -+ hdmi_modb(hdmi, n, AUDPKT_ACR_N_VALUE, AUDPKT_ACR_CONTROL0); -+ -+ /* Set CTS */ -+ if (cts) -+ hdmi_modb(hdmi, AUDPKT_ACR_CTS_OVR_EN, AUDPKT_ACR_CTS_OVR_EN_MSK, -+ AUDPKT_ACR_CONTROL1); -+ else -+ hdmi_modb(hdmi, 0, AUDPKT_ACR_CTS_OVR_EN_MSK, -+ AUDPKT_ACR_CONTROL1); -+ -+ hdmi_modb(hdmi, AUDPKT_ACR_CTS_OVR_VAL(cts), AUDPKT_ACR_CTS_OVR_VAL_MSK, -+ AUDPKT_ACR_CONTROL1); -+} -+ -+static int hdmi_match_tmds_n_table(struct dw_hdmi_qp *hdmi, -+ unsigned long pixel_clk, -+ unsigned long freq) -+{ -+ const struct dw_hdmi_plat_data *plat_data = hdmi->plat_data; -+ const struct dw_hdmi_audio_tmds_n *tmds_n = NULL; -+ int i; -+ -+ if (plat_data->tmds_n_table) { -+ for (i = 0; plat_data->tmds_n_table[i].tmds != 0; i++) { -+ if (pixel_clk == plat_data->tmds_n_table[i].tmds) { -+ tmds_n = &plat_data->tmds_n_table[i]; -+ break; -+ } -+ } -+ } -+ -+ if (tmds_n == NULL) { -+ for (i = 0; common_tmds_n_table[i].tmds != 0; i++) { -+ if (pixel_clk == common_tmds_n_table[i].tmds) { -+ tmds_n = &common_tmds_n_table[i]; -+ break; -+ } -+ } -+ } -+ -+ if (tmds_n == NULL) -+ return -ENOENT; -+ -+ switch (freq) { -+ case 32000: -+ return tmds_n->n_32k; -+ case 44100: -+ case 88200: -+ case 176400: -+ return (freq / 44100) * tmds_n->n_44k1; -+ case 48000: -+ case 96000: -+ case 192000: -+ return (freq / 48000) * tmds_n->n_48k; -+ default: -+ return -ENOENT; -+ } -+} -+ -+static u64 hdmi_audio_math_diff(unsigned int freq, unsigned int n, -+ unsigned int pixel_clk) -+{ -+ u64 final, diff; -+ u64 cts; -+ -+ final = (u64)pixel_clk * n; -+ -+ cts = final; -+ do_div(cts, 128 * freq); -+ -+ diff = final - (u64)cts * (128 * freq); -+ -+ return diff; -+} -+ -+static unsigned int hdmi_compute_n(struct dw_hdmi_qp *hdmi, -+ unsigned long pixel_clk, -+ unsigned long freq) -+{ -+ unsigned int min_n = DIV_ROUND_UP((128 * freq), 1500); -+ unsigned int max_n = (128 * freq) / 300; -+ unsigned int ideal_n = (128 * freq) / 1000; -+ unsigned int best_n_distance = ideal_n; -+ unsigned int best_n = 0; -+ u64 best_diff = U64_MAX; -+ int n; -+ -+ /* If the ideal N could satisfy the audio math, then just take it */ -+ if (hdmi_audio_math_diff(freq, ideal_n, pixel_clk) == 0) -+ return ideal_n; -+ -+ for (n = min_n; n <= max_n; n++) { -+ u64 diff = hdmi_audio_math_diff(freq, n, pixel_clk); -+ -+ if (diff < best_diff || (diff == best_diff && -+ abs(n - ideal_n) < best_n_distance)) { -+ best_n = n; -+ best_diff = diff; -+ best_n_distance = abs(best_n - ideal_n); -+ } -+ -+ /* -+ * The best N already satisfy the audio math, and also be -+ * the closest value to ideal N, so just cut the loop. -+ */ -+ if ((best_diff == 0) && (abs(n - ideal_n) > best_n_distance)) -+ break; -+ } -+ -+ return best_n; -+} -+ -+static unsigned int hdmi_find_n(struct dw_hdmi_qp *hdmi, unsigned long pixel_clk, -+ unsigned long sample_rate) -+{ -+ int n; -+ -+ n = hdmi_match_tmds_n_table(hdmi, pixel_clk, sample_rate); -+ if (n > 0) -+ return n; -+ -+ dev_warn(hdmi->dev, "Rate %lu missing; compute N dynamically\n", -+ pixel_clk); -+ -+ return hdmi_compute_n(hdmi, pixel_clk, sample_rate); -+} -+ -+/* -+ * When transmitting IEC60958 linear PCM audio, these registers allow to -+ * configure the channel status information of all the channel status -+ * bits in the IEC60958 frame. For the moment this configuration is only -+ * used when the I2S audio interface, General Purpose Audio (GPA), -+ * or AHB audio DMA (AHBAUDDMA) interface is active -+ * (for S/PDIF interface this information comes from the stream). -+ */ -+void dw_hdmi_qp_set_channel_status(struct dw_hdmi_qp *hdmi, -+ u8 *channel_status) -+{ -+ /* Set channel status */ -+ hdmi_writel(hdmi, channel_status[3] | (channel_status[4] << 8), -+ AUDPKT_CHSTATUS_OVR1); -+ hdmi_modb(hdmi, AUDPKT_CHSTATUS_OVR_EN, -+ AUDPKT_CHSTATUS_OVR_EN_MASK, AUDPKT_CONTROL0); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_channel_status); -+ -+static void hdmi_set_clk_regenerator(struct dw_hdmi_qp *hdmi, -+ unsigned long pixel_clk, unsigned int sample_rate) -+{ -+ unsigned int n = 0, cts = 0; -+ -+ n = hdmi_find_n(hdmi, pixel_clk, sample_rate); -+ -+ spin_lock_irq(&hdmi->audio_lock); -+ hdmi->audio_n = n; -+ hdmi->audio_cts = cts; -+ hdmi_set_cts_n(hdmi, cts, hdmi->audio_enable ? n : 0); -+ spin_unlock_irq(&hdmi->audio_lock); -+} -+ -+static void hdmi_init_clk_regenerator(struct dw_hdmi_qp *hdmi) -+{ -+ mutex_lock(&hdmi->audio_mutex); -+ hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate); -+ mutex_unlock(&hdmi->audio_mutex); -+} -+ -+static void hdmi_clk_regenerator_update_pixel_clock(struct dw_hdmi_qp *hdmi) -+{ -+ mutex_lock(&hdmi->audio_mutex); -+ hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock, -+ hdmi->sample_rate); -+ mutex_unlock(&hdmi->audio_mutex); -+} -+ -+void dw_hdmi_qp_set_sample_rate(struct dw_hdmi_qp *hdmi, unsigned int rate) -+{ -+ mutex_lock(&hdmi->audio_mutex); -+ hdmi->sample_rate = rate; -+ hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mtmdsclock, -+ hdmi->sample_rate); -+ mutex_unlock(&hdmi->audio_mutex); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_sample_rate); -+ -+void dw_hdmi_qp_set_channel_count(struct dw_hdmi_qp *hdmi, unsigned int cnt) -+{ -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_channel_count); -+ -+void dw_hdmi_qp_set_channel_allocation(struct dw_hdmi_qp *hdmi, unsigned int ca) -+{ -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_channel_allocation); -+ -+static void hdmi_enable_audio_clk(struct dw_hdmi_qp *hdmi, bool enable) -+{ -+ if (enable) -+ hdmi_modb(hdmi, 0, -+ AVP_DATAPATH_PACKET_AUDIO_SWDISABLE, GLOBAL_SWDISABLE); -+ else -+ hdmi_modb(hdmi, AVP_DATAPATH_PACKET_AUDIO_SWDISABLE, -+ AVP_DATAPATH_PACKET_AUDIO_SWDISABLE, GLOBAL_SWDISABLE); -+} -+ -+static void dw_hdmi_i2s_audio_enable(struct dw_hdmi_qp *hdmi) -+{ -+ hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n); -+ hdmi_enable_audio_clk(hdmi, true); -+} -+ -+static void dw_hdmi_i2s_audio_disable(struct dw_hdmi_qp *hdmi) -+{ -+ hdmi_enable_audio_clk(hdmi, false); -+} -+ -+void dw_hdmi_qp_audio_enable(struct dw_hdmi_qp *hdmi) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hdmi->audio_lock, flags); -+ hdmi->audio_enable = true; -+ if (hdmi->enable_audio) -+ hdmi->enable_audio(hdmi); -+ spin_unlock_irqrestore(&hdmi->audio_lock, flags); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_audio_enable); -+ -+void dw_hdmi_qp_audio_disable(struct dw_hdmi_qp *hdmi) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&hdmi->audio_lock, flags); -+ hdmi->audio_enable = false; -+ if (hdmi->disable_audio) -+ hdmi->disable_audio(hdmi); -+ spin_unlock_irqrestore(&hdmi->audio_lock, flags); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_audio_disable); -+ -+static bool hdmi_bus_fmt_is_rgb(unsigned int bus_format) -+{ -+ switch (bus_format) { -+ case MEDIA_BUS_FMT_RGB888_1X24: -+ case MEDIA_BUS_FMT_RGB101010_1X30: -+ case MEDIA_BUS_FMT_RGB121212_1X36: -+ case MEDIA_BUS_FMT_RGB161616_1X48: -+ return true; -+ -+ default: -+ return false; -+ } -+} -+ -+static bool hdmi_bus_fmt_is_yuv444(unsigned int bus_format) -+{ -+ switch (bus_format) { -+ case MEDIA_BUS_FMT_YUV8_1X24: -+ case MEDIA_BUS_FMT_YUV10_1X30: -+ case MEDIA_BUS_FMT_YUV12_1X36: -+ case MEDIA_BUS_FMT_YUV16_1X48: -+ return true; -+ -+ default: -+ return false; -+ } -+} -+ -+static bool hdmi_bus_fmt_is_yuv422(unsigned int bus_format) -+{ -+ switch (bus_format) { -+ case MEDIA_BUS_FMT_UYVY8_1X16: -+ case MEDIA_BUS_FMT_UYVY10_1X20: -+ case MEDIA_BUS_FMT_UYVY12_1X24: -+ return true; -+ -+ default: -+ return false; -+ } -+} -+ -+static bool hdmi_bus_fmt_is_yuv420(unsigned int bus_format) -+{ -+ switch (bus_format) { -+ case MEDIA_BUS_FMT_UYYVYY8_0_5X24: -+ case MEDIA_BUS_FMT_UYYVYY10_0_5X30: -+ case MEDIA_BUS_FMT_UYYVYY12_0_5X36: -+ case MEDIA_BUS_FMT_UYYVYY16_0_5X48: -+ return true; -+ -+ default: -+ return false; -+ } -+} -+ -+static int hdmi_bus_fmt_color_depth(unsigned int bus_format) -+{ -+ switch (bus_format) { -+ case MEDIA_BUS_FMT_RGB888_1X24: -+ case MEDIA_BUS_FMT_YUV8_1X24: -+ case MEDIA_BUS_FMT_UYVY8_1X16: -+ case MEDIA_BUS_FMT_UYYVYY8_0_5X24: -+ return 8; -+ -+ case MEDIA_BUS_FMT_RGB101010_1X30: -+ case MEDIA_BUS_FMT_YUV10_1X30: -+ case MEDIA_BUS_FMT_UYVY10_1X20: -+ case MEDIA_BUS_FMT_UYYVYY10_0_5X30: -+ return 10; -+ -+ case MEDIA_BUS_FMT_RGB121212_1X36: -+ case MEDIA_BUS_FMT_YUV12_1X36: -+ case MEDIA_BUS_FMT_UYVY12_1X24: -+ case MEDIA_BUS_FMT_UYYVYY12_0_5X36: -+ return 12; -+ -+ case MEDIA_BUS_FMT_RGB161616_1X48: -+ case MEDIA_BUS_FMT_YUV16_1X48: -+ case MEDIA_BUS_FMT_UYYVYY16_0_5X48: -+ return 16; -+ -+ default: -+ return 0; -+ } -+} -+ -+static void dw_hdmi_i2c_init(struct dw_hdmi_qp *hdmi) -+{ -+ /* Software reset */ -+ hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); -+ hdmi_writel(hdmi, 0x00, I2CM_CONTROL0); -+ -+ hdmi_writel(hdmi, 0x085c085c, I2CM_FM_SCL_CONFIG0); -+ -+ hdmi_modb(hdmi, 0, I2CM_FM_EN, I2CM_INTERFACE_CONTROL0); -+ -+ /* Clear DONE and ERROR interrupts */ -+ hdmi_writel(hdmi, I2CM_OP_DONE_CLEAR | I2CM_NACK_RCVD_CLEAR, -+ MAINUNIT_1_INT_CLEAR); -+} -+ -+static int dw_hdmi_i2c_read(struct dw_hdmi_qp *hdmi, -+ unsigned char *buf, unsigned int length) -+{ -+ struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; -+ int stat; -+ -+ if (!i2c->is_regaddr) { -+ dev_dbg(hdmi->dev, "set read register address to 0\n"); -+ i2c->slave_reg = 0x00; -+ i2c->is_regaddr = true; -+ } -+ -+ while (length--) { -+ reinit_completion(&i2c->cmp); -+ -+ hdmi_modb(hdmi, i2c->slave_reg++ << 12, I2CM_ADDR, -+ I2CM_INTERFACE_CONTROL0); -+ -+ hdmi_modb(hdmi, I2CM_FM_READ, I2CM_WR_MASK, -+ I2CM_INTERFACE_CONTROL0); -+ -+ stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10); -+ if (!stat) { -+ dev_err(hdmi->dev, "i2c read time out!\n"); -+ hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); -+ return -EAGAIN; -+ } -+ -+ /* Check for error condition on the bus */ -+ if (i2c->stat & I2CM_NACK_RCVD_IRQ) { -+ dev_err(hdmi->dev, "i2c read err!\n"); -+ hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); -+ return -EIO; -+ } -+ -+ *buf++ = hdmi_readl(hdmi, I2CM_INTERFACE_RDDATA_0_3) & 0xff; -+ dev_dbg(hdmi->dev, "i2c read done! i2c->stat:%02x 0x%02x\n", -+ i2c->stat, hdmi_readl(hdmi, I2CM_INTERFACE_RDDATA_0_3)); -+ hdmi_modb(hdmi, 0, I2CM_WR_MASK, I2CM_INTERFACE_CONTROL0); -+ } -+ i2c->is_segment = false; -+ -+ return 0; -+} -+ -+static int dw_hdmi_i2c_write(struct dw_hdmi_qp *hdmi, -+ unsigned char *buf, unsigned int length) -+{ -+ struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; -+ int stat; -+ -+ if (!i2c->is_regaddr) { -+ /* Use the first write byte as register address */ -+ i2c->slave_reg = buf[0]; -+ length--; -+ buf++; -+ i2c->is_regaddr = true; -+ } -+ -+ while (length--) { -+ reinit_completion(&i2c->cmp); -+ -+ hdmi_writel(hdmi, *buf++, I2CM_INTERFACE_WRDATA_0_3); -+ hdmi_modb(hdmi, i2c->slave_reg++ << 12, I2CM_ADDR, -+ I2CM_INTERFACE_CONTROL0); -+ hdmi_modb(hdmi, I2CM_FM_WRITE, I2CM_WR_MASK, -+ I2CM_INTERFACE_CONTROL0); -+ -+ stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10); -+ if (!stat) { -+ dev_err(hdmi->dev, "i2c write time out!\n"); -+ hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); -+ return -EAGAIN; -+ } -+ -+ /* Check for error condition on the bus */ -+ if (i2c->stat & I2CM_NACK_RCVD_IRQ) { -+ dev_err(hdmi->dev, "i2c write nack!\n"); -+ hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); -+ return -EIO; -+ } -+ hdmi_modb(hdmi, 0, I2CM_WR_MASK, I2CM_INTERFACE_CONTROL0); -+ } -+ dev_dbg(hdmi->dev, "i2c write done!\n"); -+ return 0; -+} -+ -+static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap, -+ struct i2c_msg *msgs, int num) -+{ -+ struct dw_hdmi_qp *hdmi = i2c_get_adapdata(adap); -+ struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; -+ u8 addr = msgs[0].addr; -+ int i, ret = 0; -+ -+ if (addr == DDC_CI_ADDR) -+ /* -+ * The internal I2C controller does not support the multi-byte -+ * read and write operations needed for DDC/CI. -+ * TOFIX: Blacklist the DDC/CI address until we filter out -+ * unsupported I2C operations. -+ */ -+ return -EOPNOTSUPP; -+ -+ dev_dbg(hdmi->dev, "i2c xfer: num: %d, addr: %#x\n", num, addr); -+ -+ for (i = 0; i < num; i++) { -+ if (msgs[i].len == 0) { -+ dev_err(hdmi->dev, -+ "unsupported transfer %d/%d, no data\n", -+ i + 1, num); -+ return -EOPNOTSUPP; -+ } -+ } -+ -+ mutex_lock(&i2c->lock); -+ -+ /* Unmute DONE and ERROR interrupts */ -+ hdmi_modb(hdmi, I2CM_NACK_RCVD_MASK_N | I2CM_OP_DONE_MASK_N, -+ I2CM_NACK_RCVD_MASK_N | I2CM_OP_DONE_MASK_N, -+ MAINUNIT_1_INT_MASK_N); -+ -+ /* Set slave device address taken from the first I2C message */ -+ if (addr == DDC_SEGMENT_ADDR && msgs[0].len == 1) -+ addr = DDC_ADDR; -+ -+ hdmi_modb(hdmi, addr << 5, I2CM_SLVADDR, I2CM_INTERFACE_CONTROL0); -+ -+ /* Set slave device register address on transfer */ -+ i2c->is_regaddr = false; -+ -+ /* Set segment pointer for I2C extended read mode operation */ -+ i2c->is_segment = false; -+ -+ for (i = 0; i < num; i++) { -+ dev_dbg(hdmi->dev, "xfer: num: %d/%d, len: %d, flags: %#x\n", -+ i + 1, num, msgs[i].len, msgs[i].flags); -+ -+ if (msgs[i].addr == DDC_SEGMENT_ADDR && msgs[i].len == 1) { -+ i2c->is_segment = true; -+ hdmi_modb(hdmi, DDC_SEGMENT_ADDR, I2CM_SEG_ADDR, -+ I2CM_INTERFACE_CONTROL1); -+ hdmi_modb(hdmi, *msgs[i].buf, I2CM_SEG_PTR, -+ I2CM_INTERFACE_CONTROL1); -+ } else { -+ if (msgs[i].flags & I2C_M_RD) -+ ret = dw_hdmi_i2c_read(hdmi, msgs[i].buf, -+ msgs[i].len); -+ else -+ ret = dw_hdmi_i2c_write(hdmi, msgs[i].buf, -+ msgs[i].len); -+ } -+ if (ret < 0) -+ break; -+ } -+ -+ if (!ret) -+ ret = num; -+ -+ /* Mute DONE and ERROR interrupts */ -+ hdmi_modb(hdmi, 0, I2CM_OP_DONE_MASK_N | I2CM_NACK_RCVD_MASK_N, -+ MAINUNIT_1_INT_MASK_N); -+ -+ mutex_unlock(&i2c->lock); -+ -+ return ret; -+} -+ -+static u32 dw_hdmi_i2c_func(struct i2c_adapter *adapter) -+{ -+ return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; -+} -+ -+static const struct i2c_algorithm dw_hdmi_algorithm = { -+ .master_xfer = dw_hdmi_i2c_xfer, -+ .functionality = dw_hdmi_i2c_func, -+}; -+ -+static struct i2c_adapter *dw_hdmi_i2c_adapter(struct dw_hdmi_qp *hdmi) -+{ -+ struct i2c_adapter *adap; -+ struct dw_hdmi_qp_i2c *i2c; -+ int ret; -+ -+ i2c = devm_kzalloc(hdmi->dev, sizeof(*i2c), GFP_KERNEL); -+ if (!i2c) -+ return ERR_PTR(-ENOMEM); -+ -+ mutex_init(&i2c->lock); -+ init_completion(&i2c->cmp); -+ -+ adap = &i2c->adap; -+ adap->class = I2C_CLASS_DDC; -+ adap->owner = THIS_MODULE; -+ adap->dev.parent = hdmi->dev; -+ adap->algo = &dw_hdmi_algorithm; -+ strscpy(adap->name, "ddc", sizeof(adap->name)); -+ i2c_set_adapdata(adap, hdmi); -+ -+ ret = i2c_add_adapter(adap); -+ if (ret) { -+ dev_warn(hdmi->dev, "cannot add %s I2C adapter\n", adap->name); -+ devm_kfree(hdmi->dev, i2c); -+ return ERR_PTR(ret); -+ } -+ -+ hdmi->i2c = i2c; -+ -+ dev_info(hdmi->dev, "registered %s I2C bus driver\n", adap->name); -+ -+ return adap; -+} -+ -+#define HDMI_PHY_EARC_MASK BIT(29) -+ -+int dw_hdmi_qp_set_earc(struct dw_hdmi_qp *hdmi) -+{ -+ u32 stat, ret; -+ -+ /* set hdmi phy earc mode */ -+ hdmi->phy.ops->set_mode(hdmi, hdmi->phy.data, HDMI_PHY_EARC_MASK, -+ true); -+ -+ ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, -+ &hdmi->previous_mode); -+ if (ret) -+ return ret; -+ -+ reinit_completion(&hdmi->earc_cmp); -+ -+ hdmi_modb(hdmi, EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ | -+ EARCRX_CMDC_DISCOVERY_DONE_IRQ, -+ EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ | -+ EARCRX_CMDC_DISCOVERY_DONE_IRQ, EARCRX_0_INT_MASK_N); -+ -+ /* start discovery */ -+ hdmi_modb(hdmi, EARCRX_CMDC_DISCOVERY_EN, EARCRX_CMDC_DISCOVERY_EN, -+ EARCRX_CMDC_CONTROL); -+ -+ /* -+ * The eARC TX device drives a logic-high-voltage-level -+ * pulse on the physical HPD connector pin, after -+ * at least 100 ms of low voltage level to start the -+ * eARC Discovery process. -+ */ -+ hdmi_modb(hdmi, EARCRX_CONNECTOR_HPD, EARCRX_CONNECTOR_HPD, -+ EARCRX_CMDC_CONTROL); -+ -+ stat = wait_for_completion_timeout(&hdmi->earc_cmp, HZ / 10); -+ if (!stat) -+ return -EAGAIN; -+ -+ if (hdmi->earc_intr & EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ) { -+ dev_err(hdmi->dev, "discovery timeout\n"); -+ return -ETIMEDOUT; -+ } else if (hdmi->earc_intr & EARCRX_CMDC_DISCOVERY_DONE_IRQ) { -+ dev_info(hdmi->dev, "discovery done\n"); -+ } else { -+ dev_err(hdmi->dev, "discovery failed\n"); -+ return -EINVAL; -+ } -+ -+ hdmi_writel(hdmi, 1, EARCRX_DMAC_PHY_CONTROL); -+ hdmi_modb(hdmi, EARCRX_CMDC_SWINIT_P, EARCRX_CMDC_SWINIT_P, -+ EARCRX_CMDC_CONFIG0); -+ -+ hdmi_writel(hdmi, 0xf3, EARCRX_DMAC_CONFIG); -+ hdmi_writel(hdmi, 0x63, EARCRX_DMAC_CONTROL0); -+ hdmi_writel(hdmi, 0xff, EARCRX_DMAC_CONTROL1); -+ -+ hdmi_modb(hdmi, EARCRX_XACTREAD_STOP_CFG | EARCRX_XACTREAD_RETRY_CFG | -+ EARCRX_CMDC_DSCVR_EARCVALID0_TO_DISC1 | EARCRX_CMDC_XACT_RESTART_EN, -+ EARCRX_XACTREAD_STOP_CFG | EARCRX_XACTREAD_RETRY_CFG | -+ EARCRX_CMDC_DSCVR_EARCVALID0_TO_DISC1 | EARCRX_CMDC_XACT_RESTART_EN, -+ EARCRX_CMDC_CONFIG0); -+ -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER0); -+ hdmi_writel(hdmi, 0x1b0e, EARCRX_DMAC_CHSTATUS_STREAMER1); -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER2); -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER3); -+ hdmi_writel(hdmi, 0xf2000000, EARCRX_DMAC_CHSTATUS_STREAMER4); -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER5); -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER6); -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER7); -+ hdmi_writel(hdmi, 0, EARCRX_DMAC_CHSTATUS_STREAMER8); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_earc); -+ -+/* ----------------------------------------------------------------------------- -+ * HDMI TX Setup -+ */ -+ -+static void hdmi_config_AVI(struct dw_hdmi_qp *hdmi, -+ const struct drm_connector *connector, -+ const struct drm_display_mode *mode) -+{ -+ struct hdmi_avi_infoframe frame; -+ u32 val, i, j; -+ u8 buff[17]; -+ enum hdmi_quantization_range rgb_quant_range = -+ hdmi->hdmi_data.quant_range; -+ -+ /* Initialise info frame from DRM mode */ -+ drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); -+ -+ /* -+ * Ignore monitor selectable quantization, use quantization set -+ * by the user -+ */ -+ drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, rgb_quant_range); -+ if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format)) -+ frame.colorspace = HDMI_COLORSPACE_YUV444; -+ else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) -+ frame.colorspace = HDMI_COLORSPACE_YUV422; -+ else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) -+ frame.colorspace = HDMI_COLORSPACE_YUV420; -+ else -+ frame.colorspace = HDMI_COLORSPACE_RGB; -+ -+ /* Set up colorimetry */ -+ if (!hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) { -+ switch (hdmi->hdmi_data.enc_out_encoding) { -+ case V4L2_YCBCR_ENC_601: -+ if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV601) -+ frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; -+ else -+ frame.colorimetry = HDMI_COLORIMETRY_ITU_601; -+ frame.extended_colorimetry = -+ HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; -+ break; -+ case V4L2_YCBCR_ENC_709: -+ if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_XV709) -+ frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; -+ else -+ frame.colorimetry = HDMI_COLORIMETRY_ITU_709; -+ frame.extended_colorimetry = -+ HDMI_EXTENDED_COLORIMETRY_XV_YCC_709; -+ break; -+ case V4L2_YCBCR_ENC_BT2020: -+ if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_BT2020) -+ frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; -+ else -+ frame.colorimetry = HDMI_COLORIMETRY_ITU_709; -+ frame.extended_colorimetry = -+ HDMI_EXTENDED_COLORIMETRY_BT2020; -+ break; -+ default: /* Carries no data */ -+ frame.colorimetry = HDMI_COLORIMETRY_ITU_601; -+ frame.extended_colorimetry = -+ HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; -+ break; -+ } -+ } else { -+ frame.colorimetry = HDMI_COLORIMETRY_NONE; -+ frame.extended_colorimetry = -+ HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; -+ } -+ -+ frame.scan_mode = HDMI_SCAN_MODE_NONE; -+ -+ hdmi_avi_infoframe_pack_only(&frame, buff, 17); -+ -+ /* -+ * The Designware IP uses a different byte format from standard -+ * AVI info frames, though generally the bits are in the correct -+ * bytes. -+ */ -+ -+ val = (frame.version << 8) | (frame.length << 16); -+ hdmi_writel(hdmi, val, PKT_AVI_CONTENTS0); -+ -+ for (i = 0; i < 4; i++) { -+ for (j = 0; j < 4; j++) { -+ if (i * 4 + j >= 14) -+ break; -+ if (!j) -+ val = buff[i * 4 + j + 3]; -+ val |= buff[i * 4 + j + 3] << (8 * j); -+ } -+ -+ hdmi_writel(hdmi, val, PKT_AVI_CONTENTS1 + i * 4); -+ } -+ -+ hdmi_modb(hdmi, PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN, -+ PKTSCHED_AVI_TX_EN | PKTSCHED_GCP_TX_EN, -+ PKTSCHED_PKT_EN); -+} -+ -+static void hdmi_config_CVTEM(struct dw_hdmi_qp *hdmi) -+{ -+ u8 ds_type = 0; -+ u8 sync = 1; -+ u8 vfr = 1; -+ u8 afr = 0; -+ u8 new = 1; -+ u8 end = 0; -+ u8 data_set_length = 136; -+ u8 hb1[6] = { 0x80, 0, 0, 0, 0, 0x40 }; -+ u8 *pps_body; -+ u32 val, i, reg; -+ struct drm_display_mode *mode = &hdmi->previous_mode; -+ int hsync, hfront, hback; -+ struct dw_hdmi_link_config *link_cfg; -+ void *data = hdmi->plat_data->phy_data; -+ -+ hdmi_modb(hdmi, 0, PKTSCHED_EMP_CVTEM_TX_EN, PKTSCHED_PKT_EN); -+ -+ if (hdmi->plat_data->get_link_cfg) { -+ link_cfg = hdmi->plat_data->get_link_cfg(data); -+ } else { -+ dev_err(hdmi->dev, "can't get frl link cfg\n"); -+ return; -+ } -+ -+ if (!link_cfg->dsc_mode) { -+ dev_info(hdmi->dev, "don't use dsc mode\n"); -+ return; -+ } -+ -+ pps_body = link_cfg->pps_payload; -+ -+ hsync = mode->hsync_end - mode->hsync_start; -+ hback = mode->htotal - mode->hsync_end; -+ hfront = mode->hsync_start - mode->hdisplay; -+ -+ for (i = 0; i < 6; i++) { -+ val = i << 16 | hb1[i] << 8; -+ hdmi_writel(hdmi, val, PKT0_EMP_CVTEM_CONTENTS0 + i * 0x20); -+ } -+ -+ val = new << 7 | end << 6 | ds_type << 4 | afr << 3 | -+ vfr << 2 | sync << 1; -+ hdmi_writel(hdmi, val, PKT0_EMP_CVTEM_CONTENTS1); -+ -+ val = data_set_length << 16 | pps_body[0] << 24; -+ hdmi_writel(hdmi, val, PKT0_EMP_CVTEM_CONTENTS2); -+ -+ reg = PKT0_EMP_CVTEM_CONTENTS3; -+ for (i = 1; i < 125; i++) { -+ if (reg == PKT1_EMP_CVTEM_CONTENTS0 || -+ reg == PKT2_EMP_CVTEM_CONTENTS0 || -+ reg == PKT3_EMP_CVTEM_CONTENTS0 || -+ reg == PKT4_EMP_CVTEM_CONTENTS0 || -+ reg == PKT5_EMP_CVTEM_CONTENTS0) { -+ reg += 4; -+ i--; -+ continue; -+ } -+ if (i % 4 == 1) -+ val = pps_body[i]; -+ if (i % 4 == 2) -+ val |= pps_body[i] << 8; -+ if (i % 4 == 3) -+ val |= pps_body[i] << 16; -+ if (!(i % 4)) { -+ val |= pps_body[i] << 24; -+ hdmi_writel(hdmi, val, reg); -+ reg += 4; -+ } -+ } -+ -+ val = (hfront & 0xff) << 24 | pps_body[127] << 16 | -+ pps_body[126] << 8 | pps_body[125]; -+ hdmi_writel(hdmi, val, PKT4_EMP_CVTEM_CONTENTS6); -+ -+ val = (hback & 0xff) << 24 | ((hsync >> 8) & 0xff) << 16 | -+ (hsync & 0xff) << 8 | ((hfront >> 8) & 0xff); -+ hdmi_writel(hdmi, val, PKT4_EMP_CVTEM_CONTENTS7); -+ -+ val = link_cfg->hcactive << 8 | ((hback >> 8) & 0xff); -+ hdmi_writel(hdmi, val, PKT5_EMP_CVTEM_CONTENTS1); -+ -+ for (i = PKT5_EMP_CVTEM_CONTENTS2; i <= PKT5_EMP_CVTEM_CONTENTS7; i += 4) -+ hdmi_writel(hdmi, 0, i); -+ -+ hdmi_modb(hdmi, PKTSCHED_EMP_CVTEM_TX_EN, PKTSCHED_EMP_CVTEM_TX_EN, -+ PKTSCHED_PKT_EN); -+} -+ -+static void hdmi_config_drm_infoframe(struct dw_hdmi_qp *hdmi, -+ const struct drm_connector *connector) -+{ -+ const struct drm_connector_state *conn_state = connector->state; -+ struct hdr_output_metadata *hdr_metadata; -+ struct hdmi_drm_infoframe frame; -+ u8 buffer[30]; -+ ssize_t err; -+ int i; -+ u32 val; -+ -+ if (!hdmi->plat_data->use_drm_infoframe) -+ return; -+ -+ hdmi_modb(hdmi, 0, PKTSCHED_DRMI_TX_EN, PKTSCHED_PKT_EN); -+ -+ if (!hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf) { -+ DRM_DEBUG("No need to set HDR metadata in infoframe\n"); -+ return; -+ } -+ -+ if (!conn_state->hdr_output_metadata) { -+ DRM_DEBUG("source metadata not set yet\n"); -+ return; -+ } -+ -+ hdr_metadata = (struct hdr_output_metadata *) -+ conn_state->hdr_output_metadata->data; -+ -+ if (!(hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf & -+ BIT(hdr_metadata->hdmi_metadata_type1.eotf))) { -+ DRM_ERROR("Not support EOTF %d\n", -+ hdr_metadata->hdmi_metadata_type1.eotf); -+ return; -+ } -+ -+ err = drm_hdmi_infoframe_set_hdr_metadata(&frame, conn_state); -+ if (err < 0) -+ return; -+ -+ err = hdmi_drm_infoframe_pack(&frame, buffer, sizeof(buffer)); -+ if (err < 0) { -+ dev_err(hdmi->dev, "Failed to pack drm infoframe: %zd\n", err); -+ return; -+ } -+ -+ val = (frame.version << 8) | (frame.length << 16); -+ hdmi_writel(hdmi, val, PKT_DRMI_CONTENTS0); -+ -+ for (i = 0; i <= frame.length; i++) { -+ if (i % 4 == 0) -+ val = buffer[3 + i]; -+ val |= buffer[3 + i] << ((i % 4) * 8); -+ -+ if (i % 4 == 3 || (i == (frame.length))) -+ hdmi_writel(hdmi, val, PKT_DRMI_CONTENTS1 + ((i / 4) * 4)); -+ } -+ -+ hdmi_modb(hdmi, PKTSCHED_DRMI_TX_EN, PKTSCHED_DRMI_TX_EN, PKTSCHED_PKT_EN); -+ -+ DRM_DEBUG("%s eotf %d end\n", __func__, -+ hdr_metadata->hdmi_metadata_type1.eotf); -+} -+ -+/* Filter out invalid setups to avoid configuring SCDC and scrambling */ -+static bool dw_hdmi_support_scdc(struct dw_hdmi_qp *hdmi, -+ const struct drm_display_info *display) -+{ -+ /* Disable if no DDC bus */ -+ if (!hdmi->ddc) -+ return false; -+ -+ /* Disable if SCDC is not supported, or if an HF-VSDB block is absent */ -+ if (!display->hdmi.scdc.supported || -+ !display->hdmi.scdc.scrambling.supported) -+ return false; -+ -+ /* -+ * Disable if display only support low TMDS rates and scrambling -+ * for low rates is not supported either -+ */ -+ if (!display->hdmi.scdc.scrambling.low_rates && -+ display->max_tmds_clock <= 340000) -+ return false; -+ -+ return true; -+} -+ -+static int hdmi_set_frl_mask(int frl_rate) -+{ -+ switch (frl_rate) { -+ case 48: -+ return FRL_12GBPS_4LANE; -+ case 40: -+ return FRL_10GBPS_4LANE; -+ case 32: -+ return FRL_8GBPS_4LANE; -+ case 24: -+ return FRL_6GBPS_4LANE; -+ case 18: -+ return FRL_6GBPS_3LANE; -+ case 9: -+ return FRL_3GBPS_3LANE; -+ } -+ -+ return 0; -+} -+ -+static int hdmi_start_flt(struct dw_hdmi_qp *hdmi, u8 rate) -+{ -+ u8 val; -+ u8 ffe_lv = 0; -+ int i = 0, stat; -+ -+ /* FLT_READY & FFE_LEVELS read */ -+ for (i = 0; i < 20; i++) { -+ drm_scdc_readb(hdmi->ddc, SCDC_STATUS_FLAGS_0, &val); -+ if (val & BIT(6)) -+ break; -+ msleep(20); -+ } -+ -+ if (i == 20) { -+ dev_err(hdmi->dev, "sink flt isn't ready\n"); -+ return -EINVAL; -+ } -+ -+ hdmi_modb(hdmi, SCDC_UPD_FLAGS_RD_IRQ, SCDC_UPD_FLAGS_RD_IRQ, -+ MAINUNIT_1_INT_MASK_N); -+ hdmi_modb(hdmi, SCDC_UPD_FLAGS_POLL_EN | SCDC_UPD_FLAGS_AUTO_CLR, -+ SCDC_UPD_FLAGS_POLL_EN | SCDC_UPD_FLAGS_AUTO_CLR, -+ SCDC_CONFIG0); -+ -+ /* max ffe level 3 */ -+ val = 3 << 4 | hdmi_set_frl_mask(rate); -+ drm_scdc_writeb(hdmi->ddc, 0x31, val); -+ -+ /* select FRL_RATE & FFE_LEVELS */ -+ hdmi_writel(hdmi, ffe_lv, FLT_CONFIG0); -+ -+ /* Start LTS_3 state in source DUT */ -+ reinit_completion(&hdmi->flt_cmp); -+ hdmi_modb(hdmi, FLT_EXIT_TO_LTSP_IRQ, FLT_EXIT_TO_LTSP_IRQ, -+ MAINUNIT_1_INT_MASK_N); -+ hdmi_writel(hdmi, 1, FLT_CONTROL0); -+ -+ /* wait for completed link training at source side */ -+ stat = wait_for_completion_timeout(&hdmi->flt_cmp, HZ * 2); -+ if (!stat) { -+ dev_err(hdmi->dev, "wait lts3 finish time out\n"); -+ hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_POLL_EN | -+ SCDC_UPD_FLAGS_AUTO_CLR, SCDC_CONFIG0); -+ hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_RD_IRQ, -+ MAINUNIT_1_INT_MASK_N); -+ return -EAGAIN; -+ } -+ -+ if (!(hdmi->flt_intr & FLT_EXIT_TO_LTSP_IRQ)) { -+ dev_err(hdmi->dev, "not to ltsp\n"); -+ hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_POLL_EN | -+ SCDC_UPD_FLAGS_AUTO_CLR, SCDC_CONFIG0); -+ hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_RD_IRQ, -+ MAINUNIT_1_INT_MASK_N); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+#define HDMI_MODE_FRL_MASK BIT(30) -+ -+static void hdmi_set_op_mode(struct dw_hdmi_qp *hdmi, -+ struct dw_hdmi_link_config *link_cfg, -+ const struct drm_connector *connector) -+{ -+ int frl_rate; -+ -+ hdmi_writel(hdmi, 0, FLT_CONFIG0); -+ if (dw_hdmi_support_scdc(hdmi, &connector->display_info)) -+ drm_scdc_writeb(hdmi->ddc, 0x31, 0); -+ msleep(20); -+ if (!link_cfg->frl_mode) { -+ dev_info(hdmi->dev, "dw hdmi qp use tmds mode\n"); -+ hdmi_modb(hdmi, 0, OPMODE_FRL, LINK_CONFIG0); -+ hdmi_modb(hdmi, 0, OPMODE_FRL_4LANES, LINK_CONFIG0); -+ return; -+ } -+ -+ if (link_cfg->frl_lanes == 4) -+ hdmi_modb(hdmi, OPMODE_FRL_4LANES, OPMODE_FRL_4LANES, -+ LINK_CONFIG0); -+ else -+ hdmi_modb(hdmi, 0, OPMODE_FRL_4LANES, LINK_CONFIG0); -+ -+ hdmi_modb(hdmi, 1, OPMODE_FRL, LINK_CONFIG0); -+ -+ frl_rate = link_cfg->frl_lanes * link_cfg->rate_per_lane; -+ hdmi_start_flt(hdmi, frl_rate); -+} -+ -+static unsigned long -+hdmi_get_tmdsclock(struct dw_hdmi_qp *hdmi, unsigned long mpixelclock) -+{ -+ unsigned long tmdsclock = mpixelclock; -+ unsigned int depth = -+ hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); -+ -+ if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) { -+ switch (depth) { -+ case 16: -+ tmdsclock = mpixelclock * 2; -+ break; -+ case 12: -+ tmdsclock = mpixelclock * 3 / 2; -+ break; -+ case 10: -+ tmdsclock = mpixelclock * 5 / 4; -+ break; -+ default: -+ break; -+ } -+ } -+ -+ return tmdsclock; -+} -+ -+static int dw_hdmi_qp_setup(struct dw_hdmi_qp *hdmi, -+ const struct drm_connector *connector, -+ const struct drm_display_mode *mode) -+{ -+ int ret; -+ void *data = hdmi->plat_data->phy_data; -+ struct hdmi_vmode_qp *vmode = &hdmi->hdmi_data.video_mode; -+ struct dw_hdmi_link_config *link_cfg; -+ u8 bytes = 0; -+ -+ hdmi->vic = drm_match_cea_mode(mode); -+ -+ if (!hdmi->vic) -+ dev_dbg(hdmi->dev, "Non-CEA mode used in HDMI\n"); -+ else -+ dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic); -+ -+ if (hdmi->plat_data->get_enc_out_encoding) -+ hdmi->hdmi_data.enc_out_encoding = -+ hdmi->plat_data->get_enc_out_encoding(data); -+ else if ((hdmi->vic == 6) || (hdmi->vic == 7) || -+ (hdmi->vic == 21) || (hdmi->vic == 22) || -+ (hdmi->vic == 2) || (hdmi->vic == 3) || -+ (hdmi->vic == 17) || (hdmi->vic == 18)) -+ hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601; -+ else -+ hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709; -+ -+ if (mode->flags & DRM_MODE_FLAG_DBLCLK) { -+ hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1; -+ hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1; -+ } else { -+ hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; -+ hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; -+ } -+ /* Get input format from plat data or fallback to RGB888 */ -+ if (hdmi->plat_data->get_input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->get_input_bus_format(data); -+ else if (hdmi->plat_data->input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->input_bus_format; -+ else -+ hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24; -+ -+ /* Default to RGB888 output format */ -+ if (hdmi->plat_data->get_output_bus_format) -+ hdmi->hdmi_data.enc_out_bus_format = -+ hdmi->plat_data->get_output_bus_format(data); -+ else -+ hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24; -+ -+ /* Get input encoding from plat data or fallback to none */ -+ if (hdmi->plat_data->get_enc_in_encoding) -+ hdmi->hdmi_data.enc_in_encoding = -+ hdmi->plat_data->get_enc_in_encoding(data); -+ else if (hdmi->plat_data->input_bus_encoding) -+ hdmi->hdmi_data.enc_in_encoding = -+ hdmi->plat_data->input_bus_encoding; -+ else -+ hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT; -+ -+ if (hdmi->plat_data->get_quant_range) -+ hdmi->hdmi_data.quant_range = -+ hdmi->plat_data->get_quant_range(data); -+ else -+ hdmi->hdmi_data.quant_range = HDMI_QUANTIZATION_RANGE_DEFAULT; -+ -+ if (hdmi->plat_data->get_link_cfg) -+ link_cfg = hdmi->plat_data->get_link_cfg(data); -+ else -+ return -EINVAL; -+ -+ hdmi->phy.ops->set_mode(hdmi, hdmi->phy.data, HDMI_MODE_FRL_MASK, -+ link_cfg->frl_mode); -+ -+ /* -+ * According to the dw-hdmi specification 6.4.2 -+ * vp_pr_cd[3:0]: -+ * 0000b: No pixel repetition (pixel sent only once) -+ * 0001b: Pixel sent two times (pixel repeated once) -+ */ -+ hdmi->hdmi_data.pix_repet_factor = -+ (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 1 : 0; -+ hdmi->hdmi_data.video_mode.mdataenablepolarity = true; -+ -+ vmode->previous_pixelclock = vmode->mpixelclock; -+ vmode->mpixelclock = mode->crtc_clock * 1000; -+ if ((mode->flags & DRM_MODE_FLAG_3D_MASK) == DRM_MODE_FLAG_3D_FRAME_PACKING) -+ vmode->mpixelclock *= 2; -+ dev_dbg(hdmi->dev, "final pixclk = %ld\n", vmode->mpixelclock); -+ vmode->previous_tmdsclock = vmode->mtmdsclock; -+ vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, vmode->mpixelclock); -+ if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) -+ vmode->mtmdsclock /= 2; -+ dev_info(hdmi->dev, "final tmdsclk = %d\n", vmode->mtmdsclock); -+ -+ ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, &hdmi->previous_mode); -+ if (ret) -+ return ret; -+ -+ if (hdmi->sink_has_audio) { -+ dev_dbg(hdmi->dev, "sink has audio support\n"); -+ -+ /* HDMI Initialization Step E - Configure audio */ -+ hdmi_clk_regenerator_update_pixel_clock(hdmi); -+ hdmi_enable_audio_clk(hdmi, hdmi->audio_enable); -+ } -+ -+ /* not for DVI mode */ -+ if (hdmi->sink_is_hdmi) { -+ dev_dbg(hdmi->dev, "%s HDMI mode\n", __func__); -+ hdmi_modb(hdmi, 0, OPMODE_DVI, LINK_CONFIG0); -+ hdmi_modb(hdmi, HDCP2_BYPASS, HDCP2_BYPASS, HDCP2LOGIC_CONFIG0); -+ if (!link_cfg->frl_mode) { -+ if (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK) { -+ drm_scdc_readb(hdmi->ddc, SCDC_SINK_VERSION, &bytes); -+ drm_scdc_writeb(hdmi->ddc, SCDC_SOURCE_VERSION, -+ min_t(u8, bytes, SCDC_MIN_SOURCE_VERSION)); -+ drm_scdc_set_high_tmds_clock_ratio(hdmi->ddc, 1); -+ drm_scdc_set_scrambling(hdmi->ddc, 1); -+ hdmi_writel(hdmi, 1, SCRAMB_CONFIG0); -+ } else { -+ if (dw_hdmi_support_scdc(hdmi, &connector->display_info)) { -+ drm_scdc_set_high_tmds_clock_ratio(hdmi->ddc, 0); -+ drm_scdc_set_scrambling(hdmi->ddc, 0); -+ } -+ hdmi_writel(hdmi, 0, SCRAMB_CONFIG0); -+ } -+ } -+ /* HDMI Initialization Step F - Configure AVI InfoFrame */ -+ hdmi_config_AVI(hdmi, connector, mode); -+ hdmi_config_CVTEM(hdmi); -+ hdmi_config_drm_infoframe(hdmi, connector); -+ hdmi_set_op_mode(hdmi, link_cfg, connector); -+ } else { -+ hdmi_modb(hdmi, OPMODE_DVI, OPMODE_DVI, LINK_CONFIG0); -+ dev_info(hdmi->dev, "%s DVI mode\n", __func__); -+ } -+ -+ return 0; -+} -+ -+static enum drm_connector_status -+dw_hdmi_connector_detect(struct drm_connector *connector, bool force) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ -+ mutex_lock(&hdmi->mutex); -+ hdmi->force = DRM_FORCE_UNSPECIFIED; -+ mutex_unlock(&hdmi->mutex); -+ -+ return hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); -+} -+ -+static int -+dw_hdmi_update_hdr_property(struct drm_connector *connector) -+{ -+ struct drm_device *dev = connector->dev; -+ struct dw_hdmi_qp *hdmi = container_of(connector, struct dw_hdmi_qp, -+ connector); -+ void *data = hdmi->plat_data->phy_data; -+ const struct hdr_static_metadata *metadata = -+ &connector->hdr_sink_metadata.hdmi_type1; -+ size_t size = sizeof(*metadata); -+ struct drm_property *property; -+ struct drm_property_blob *blob; -+ int ret; -+ -+ if (hdmi->plat_data->get_hdr_property) -+ property = hdmi->plat_data->get_hdr_property(data); -+ else -+ return -EINVAL; -+ -+ if (hdmi->plat_data->get_hdr_blob) -+ blob = hdmi->plat_data->get_hdr_blob(data); -+ else -+ return -EINVAL; -+ -+ ret = drm_property_replace_global_blob(dev, &blob, size, metadata, -+ &connector->base, property); -+ return ret; -+} -+ -+static int dw_hdmi_connector_get_modes(struct drm_connector *connector) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ struct hdr_static_metadata *metedata = -+ &connector->hdr_sink_metadata.hdmi_type1; -+ struct edid *edid; -+ struct drm_display_mode *mode; -+ struct drm_display_info *info = &connector->display_info; -+ void *data = hdmi->plat_data->phy_data; -+ int i, ret = 0; -+ -+ if (!hdmi->ddc) -+ return 0; -+ -+ memset(metedata, 0, sizeof(*metedata)); -+ edid = drm_get_edid(connector, hdmi->ddc); -+ if (edid) { -+ dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", -+ edid->width_cm, edid->height_cm); -+ -+ hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid); -+ hdmi->sink_has_audio = drm_detect_monitor_audio(edid); -+ drm_connector_update_edid_property(connector, edid); -+ cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid); -+ if (hdmi->plat_data->get_edid_dsc_info) -+ hdmi->plat_data->get_edid_dsc_info(data, edid); -+ ret = drm_add_edid_modes(connector, edid); -+ dw_hdmi_update_hdr_property(connector); -+ kfree(edid); -+ } else { -+ hdmi->sink_is_hdmi = true; -+ hdmi->sink_has_audio = true; -+ -+ for (i = 0; i < ARRAY_SIZE(dw_hdmi_default_modes); i++) { -+ const struct drm_display_mode *ptr = -+ &dw_hdmi_default_modes[i]; -+ -+ mode = drm_mode_duplicate(connector->dev, ptr); -+ if (mode) { -+ if (!i) { -+ mode->type = DRM_MODE_TYPE_PREFERRED; -+ mode->picture_aspect_ratio = -+ HDMI_PICTURE_ASPECT_NONE; -+ } -+ drm_mode_probed_add(connector, mode); -+ ret++; -+ } -+ } -+ info->edid_hdmi_dc_modes = 0; -+ info->hdmi.y420_dc_modes = 0; -+ info->color_formats = 0; -+ -+ dev_info(hdmi->dev, "failed to get edid\n"); -+ } -+ -+ return ret; -+} -+ -+static int -+dw_hdmi_atomic_connector_set_property(struct drm_connector *connector, -+ struct drm_connector_state *state, -+ struct drm_property *property, -+ uint64_t val) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops; -+ -+ if (ops && ops->set_property) -+ return ops->set_property(connector, state, property, -+ val, hdmi->plat_data->phy_data); -+ else -+ return -EINVAL; -+} -+ -+static int -+dw_hdmi_atomic_connector_get_property(struct drm_connector *connector, -+ const struct drm_connector_state *state, -+ struct drm_property *property, -+ uint64_t *val) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ const struct dw_hdmi_property_ops *ops = hdmi->plat_data->property_ops; -+ -+ if (ops && ops->get_property) -+ return ops->get_property(connector, state, property, -+ val, hdmi->plat_data->phy_data); -+ else -+ return -EINVAL; -+} -+ -+static int -+dw_hdmi_connector_set_property(struct drm_connector *connector, -+ struct drm_property *property, uint64_t val) -+{ -+ return dw_hdmi_atomic_connector_set_property(connector, NULL, -+ property, val); -+} -+ -+static void dw_hdmi_attach_properties(struct dw_hdmi_qp *hdmi) -+{ -+ unsigned int color = MEDIA_BUS_FMT_RGB888_1X24; -+ const struct dw_hdmi_property_ops *ops = -+ hdmi->plat_data->property_ops; -+ -+ if (ops && ops->attach_properties) -+ return ops->attach_properties(&hdmi->connector, color, 0, -+ hdmi->plat_data->phy_data); -+} -+ -+static void dw_hdmi_destroy_properties(struct dw_hdmi_qp *hdmi) -+{ -+ const struct dw_hdmi_property_ops *ops = -+ hdmi->plat_data->property_ops; -+ -+ if (ops && ops->destroy_properties) -+ return ops->destroy_properties(&hdmi->connector, -+ hdmi->plat_data->phy_data); -+} -+ -+static struct drm_encoder * -+dw_hdmi_connector_best_encoder(struct drm_connector *connector) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ -+ return hdmi->bridge.encoder; -+} -+ -+static bool dw_hdmi_color_changed(struct drm_connector *connector) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ void *data = hdmi->plat_data->phy_data; -+ bool ret = false; -+ -+ if (hdmi->plat_data->get_color_changed) -+ ret = hdmi->plat_data->get_color_changed(data); -+ -+ return ret; -+} -+ -+static bool hdr_metadata_equal(const struct drm_connector_state *old_state, -+ const struct drm_connector_state *new_state) -+{ -+ struct drm_property_blob *old_blob = old_state->hdr_output_metadata; -+ struct drm_property_blob *new_blob = new_state->hdr_output_metadata; -+ -+ if (!old_blob || !new_blob) -+ return old_blob == new_blob; -+ -+ if (old_blob->length != new_blob->length) -+ return false; -+ -+ return !memcmp(old_blob->data, new_blob->data, old_blob->length); -+} -+ -+static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, -+ struct drm_atomic_state *state) -+{ -+ struct drm_connector_state *old_state = -+ drm_atomic_get_old_connector_state(state, connector); -+ struct drm_connector_state *new_state = -+ drm_atomic_get_new_connector_state(state, connector); -+ struct drm_crtc *crtc = new_state->crtc; -+ struct drm_crtc_state *crtc_state; -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ struct drm_display_mode *mode = NULL; -+ void *data = hdmi->plat_data->phy_data; -+ struct hdmi_vmode_qp *vmode = &hdmi->hdmi_data.video_mode; -+ unsigned int in_bus_format = hdmi->hdmi_data.enc_in_bus_format; -+ unsigned int out_bus_format = hdmi->hdmi_data.enc_out_bus_format; -+ bool color_changed = false; -+ -+ if (!crtc) -+ return 0; -+ -+ crtc_state = drm_atomic_get_crtc_state(state, crtc); -+ if (IS_ERR(crtc_state)) -+ return PTR_ERR(crtc_state); -+ -+ /* -+ * If HDMI is enabled in uboot, it's need to record -+ * drm_display_mode and set phy status to enabled. -+ */ -+ if (!vmode->mpixelclock) { -+ crtc_state = drm_atomic_get_crtc_state(state, crtc); -+ if (hdmi->plat_data->get_enc_in_encoding) -+ hdmi->hdmi_data.enc_in_encoding = -+ hdmi->plat_data->get_enc_in_encoding(data); -+ if (hdmi->plat_data->get_enc_out_encoding) -+ hdmi->hdmi_data.enc_out_encoding = -+ hdmi->plat_data->get_enc_out_encoding(data); -+ if (hdmi->plat_data->get_input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->get_input_bus_format(data); -+ if (hdmi->plat_data->get_output_bus_format) -+ hdmi->hdmi_data.enc_out_bus_format = -+ hdmi->plat_data->get_output_bus_format(data); -+ -+ mode = &crtc_state->mode; -+ memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); -+ vmode->mpixelclock = mode->crtc_clock * 1000; -+ vmode->previous_pixelclock = mode->clock; -+ vmode->previous_tmdsclock = mode->clock; -+ vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, -+ vmode->mpixelclock); -+ if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) -+ vmode->mtmdsclock /= 2; -+ -+ if (in_bus_format != hdmi->hdmi_data.enc_in_bus_format || -+ out_bus_format != hdmi->hdmi_data.enc_out_bus_format) -+ color_changed = true; -+ } -+ -+ if (!hdr_metadata_equal(old_state, new_state) || -+ dw_hdmi_color_changed(connector) || color_changed) { -+ crtc_state = drm_atomic_get_crtc_state(state, crtc); -+ if (IS_ERR(crtc_state)) -+ return PTR_ERR(crtc_state); -+ -+ crtc_state->mode_changed = true; -+ } -+ -+ return 0; -+} -+ -+static void dw_hdmi_connector_force(struct drm_connector *connector) -+{ -+ struct dw_hdmi_qp *hdmi = -+ container_of(connector, struct dw_hdmi_qp, connector); -+ -+ mutex_lock(&hdmi->mutex); -+ -+ if (hdmi->force != connector->force) { -+ if (!hdmi->disabled && connector->force == DRM_FORCE_OFF) -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, -+ false); -+ else if (hdmi->disabled && connector->force == DRM_FORCE_ON) -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, -+ true); -+ } -+ -+ hdmi->force = connector->force; -+ mutex_unlock(&hdmi->mutex); -+} -+ -+static int dw_hdmi_qp_fill_modes(struct drm_connector *connector, u32 max_x, -+ u32 max_y) -+{ -+ return drm_helper_probe_single_connector_modes(connector, 9000, 9000); -+} -+ -+static const struct drm_connector_funcs dw_hdmi_connector_funcs = { -+ .fill_modes = dw_hdmi_qp_fill_modes, -+ .detect = dw_hdmi_connector_detect, -+ .destroy = drm_connector_cleanup, -+ .force = dw_hdmi_connector_force, -+ .reset = drm_atomic_helper_connector_reset, -+ .set_property = dw_hdmi_connector_set_property, -+ .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, -+ .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, -+ .atomic_set_property = dw_hdmi_atomic_connector_set_property, -+ .atomic_get_property = dw_hdmi_atomic_connector_get_property, -+}; -+ -+static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = { -+ .get_modes = dw_hdmi_connector_get_modes, -+ .best_encoder = dw_hdmi_connector_best_encoder, -+ .atomic_check = dw_hdmi_connector_atomic_check, -+}; -+ -+static int dw_hdmi_qp_bridge_attach(struct drm_bridge *bridge, -+ enum drm_bridge_attach_flags flags) -+{ -+ struct dw_hdmi_qp *hdmi = bridge->driver_private; -+ struct drm_encoder *encoder = bridge->encoder; -+ struct drm_connector *connector = &hdmi->connector; -+ struct cec_connector_info conn_info; -+ struct cec_notifier *notifier; -+ -+ connector->interlace_allowed = 1; -+ connector->polled = DRM_CONNECTOR_POLL_HPD; -+ -+ drm_connector_helper_add(connector, &dw_hdmi_connector_helper_funcs); -+ -+ drm_connector_init(bridge->dev, connector, &dw_hdmi_connector_funcs, -+ DRM_MODE_CONNECTOR_HDMIA); -+ -+ drm_connector_attach_encoder(connector, encoder); -+ dw_hdmi_attach_properties(hdmi); -+ -+ cec_fill_conn_info_from_drm(&conn_info, connector); -+ notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info); -+ if (!notifier) -+ return -ENOMEM; -+ -+ mutex_lock(&hdmi->cec_notifier_mutex); -+ hdmi->cec_notifier = notifier; -+ mutex_unlock(&hdmi->cec_notifier_mutex); -+ -+ return 0; -+} -+ -+static void dw_hdmi_qp_bridge_detach(struct drm_bridge *bridge) -+{ -+ struct dw_hdmi_qp *hdmi = bridge->driver_private; -+ -+ mutex_lock(&hdmi->cec_notifier_mutex); -+ cec_notifier_conn_unregister(hdmi->cec_notifier); -+ hdmi->cec_notifier = NULL; -+ mutex_unlock(&hdmi->cec_notifier_mutex); -+} -+ -+static enum drm_mode_status -+dw_hdmi_qp_bridge_mode_valid(struct drm_bridge *bridge, -+ const struct drm_display_info *info, -+ const struct drm_display_mode *mode) -+{ -+ struct dw_hdmi_qp *hdmi = bridge->driver_private; -+ struct drm_connector *connector = &hdmi->connector; -+ const struct dw_hdmi_plat_data *pdata = hdmi->plat_data; -+ enum drm_mode_status mode_status = MODE_OK; -+ -+ if (pdata->mode_valid) -+ mode_status = pdata->mode_valid(connector, pdata->priv_data, -+ info, mode); -+ -+ return mode_status; -+} -+ -+static void dw_hdmi_qp_bridge_mode_set(struct drm_bridge *bridge, -+ const struct drm_display_mode *orig_mode, -+ const struct drm_display_mode *mode) -+{ -+ struct dw_hdmi_qp *hdmi = bridge->driver_private; -+ -+ mutex_lock(&hdmi->mutex); -+ -+ /* Store the display mode for plugin/DKMS poweron events */ -+ memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); -+ -+ mutex_unlock(&hdmi->mutex); -+} -+ -+static void dw_hdmi_qp_bridge_atomic_disable(struct drm_bridge *bridge, -+ struct drm_bridge_state *old_state) -+{ -+ struct dw_hdmi_qp *hdmi = bridge->driver_private; -+ -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, false); -+ handle_plugged_change(hdmi, false); -+ mutex_lock(&hdmi->mutex); -+ hdmi->disabled = true; -+ hdmi->curr_conn = NULL; -+ hdmi_writel(hdmi, 0, PKTSCHED_PKT_EN); -+ if (hdmi->phy.ops->disable) -+ hdmi->phy.ops->disable(hdmi, hdmi->phy.data); -+ mutex_unlock(&hdmi->mutex); -+} -+ -+static void dw_hdmi_qp_bridge_atomic_enable(struct drm_bridge *bridge, -+ struct drm_bridge_state *old_state) -+{ -+ struct dw_hdmi_qp *hdmi = bridge->driver_private; -+ struct drm_atomic_state *state = old_state->base.state; -+ struct drm_connector *connector; -+ -+ connector = drm_atomic_get_new_connector_for_encoder(state, -+ bridge->encoder); -+ -+ mutex_lock(&hdmi->mutex); -+ hdmi->disabled = false; -+ hdmi->curr_conn = connector; -+ dw_hdmi_qp_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode); -+ mutex_unlock(&hdmi->mutex); -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, true); -+ handle_plugged_change(hdmi, true); -+} -+ -+static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = { -+ .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, -+ .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, -+ .atomic_reset = drm_atomic_helper_bridge_reset, -+ .attach = dw_hdmi_qp_bridge_attach, -+ .detach = dw_hdmi_qp_bridge_detach, -+ .mode_set = dw_hdmi_qp_bridge_mode_set, -+ .mode_valid = dw_hdmi_qp_bridge_mode_valid, -+ .atomic_enable = dw_hdmi_qp_bridge_atomic_enable, -+ .atomic_disable = dw_hdmi_qp_bridge_atomic_disable, -+}; -+ -+void dw_hdmi_qp_set_cec_adap(struct dw_hdmi_qp *hdmi, struct cec_adapter *adap) -+{ -+ hdmi->cec_adap = adap; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_set_cec_adap); -+ -+static irqreturn_t dw_hdmi_qp_main_hardirq(int irq, void *dev_id) -+{ -+ struct dw_hdmi_qp *hdmi = dev_id; -+ struct dw_hdmi_qp_i2c *i2c = hdmi->i2c; -+ u32 stat; -+ -+ stat = hdmi_readl(hdmi, MAINUNIT_1_INT_STATUS); -+ -+ i2c->stat = stat & (I2CM_OP_DONE_IRQ | I2CM_READ_REQUEST_IRQ | -+ I2CM_NACK_RCVD_IRQ); -+ hdmi->scdc_intr = stat & (SCDC_UPD_FLAGS_RD_IRQ | -+ SCDC_UPD_FLAGS_CHG_IRQ | -+ SCDC_UPD_FLAGS_CLR_IRQ | -+ SCDC_RR_REPLY_STOP_IRQ | -+ SCDC_NACK_RCVD_IRQ); -+ hdmi->flt_intr = stat & (FLT_EXIT_TO_LTSP_IRQ | -+ FLT_EXIT_TO_LTS4_IRQ | -+ FLT_EXIT_TO_LTSL_IRQ); -+ -+ dev_dbg(hdmi->dev, "i2c main unit irq:%#x\n", stat); -+ if (i2c->stat) { -+ hdmi_writel(hdmi, i2c->stat, MAINUNIT_1_INT_CLEAR); -+ complete(&i2c->cmp); -+ } -+ -+ if (hdmi->flt_intr) { -+ dev_dbg(hdmi->dev, "i2c flt irq:%#x\n", hdmi->flt_intr); -+ hdmi_writel(hdmi, hdmi->flt_intr, MAINUNIT_1_INT_CLEAR); -+ complete(&hdmi->flt_cmp); -+ } -+ -+ if (hdmi->scdc_intr) { -+ u8 val; -+ -+ dev_dbg(hdmi->dev, "i2c scdc irq:%#x\n", hdmi->scdc_intr); -+ hdmi_writel(hdmi, hdmi->scdc_intr, MAINUNIT_1_INT_CLEAR); -+ val = hdmi_readl(hdmi, SCDC_STATUS0); -+ -+ /* frl start */ -+ if (val & BIT(4)) { -+ hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_POLL_EN | -+ SCDC_UPD_FLAGS_AUTO_CLR, SCDC_CONFIG0); -+ hdmi_modb(hdmi, 0, SCDC_UPD_FLAGS_RD_IRQ, -+ MAINUNIT_1_INT_MASK_N); -+ dev_info(hdmi->dev, "frl start\n"); -+ } -+ -+ } -+ -+ if (stat) -+ return IRQ_HANDLED; -+ -+ return IRQ_NONE; -+} -+ -+static irqreturn_t dw_hdmi_qp_avp_hardirq(int irq, void *dev_id) -+{ -+ struct dw_hdmi_qp *hdmi = dev_id; -+ u32 stat; -+ -+ stat = hdmi_readl(hdmi, AVP_1_INT_STATUS); -+ if (stat) { -+ dev_dbg(hdmi->dev, "HDCP irq %#x\n", stat); -+ stat &= ~stat; -+ hdmi_writel(hdmi, stat, AVP_1_INT_MASK_N); -+ return IRQ_WAKE_THREAD; -+ } -+ -+ return IRQ_NONE; -+} -+ -+static irqreturn_t dw_hdmi_qp_earc_hardirq(int irq, void *dev_id) -+{ -+ struct dw_hdmi_qp *hdmi = dev_id; -+ u32 stat; -+ -+ stat = hdmi_readl(hdmi, EARCRX_0_INT_STATUS); -+ if (stat) { -+ dev_dbg(hdmi->dev, "earc irq %#x\n", stat); -+ stat &= ~stat; -+ hdmi_writel(hdmi, stat, EARCRX_0_INT_MASK_N); -+ return IRQ_WAKE_THREAD; -+ } -+ -+ return IRQ_NONE; -+} -+ -+static irqreturn_t dw_hdmi_qp_avp_irq(int irq, void *dev_id) -+{ -+ struct dw_hdmi_qp *hdmi = dev_id; -+ u32 stat; -+ -+ stat = hdmi_readl(hdmi, AVP_1_INT_STATUS); -+ -+ if (!stat) -+ return IRQ_NONE; -+ -+ hdmi_writel(hdmi, stat, AVP_1_INT_CLEAR); -+ -+ return IRQ_HANDLED; -+} -+ -+static irqreturn_t dw_hdmi_qp_earc_irq(int irq, void *dev_id) -+{ -+ struct dw_hdmi_qp *hdmi = dev_id; -+ u32 stat; -+ -+ stat = hdmi_readl(hdmi, EARCRX_0_INT_STATUS); -+ -+ if (!stat) -+ return IRQ_NONE; -+ -+ hdmi_writel(hdmi, stat, EARCRX_0_INT_CLEAR); -+ -+ hdmi->earc_intr = stat; -+ complete(&hdmi->earc_cmp); -+ -+ return IRQ_HANDLED; -+} -+ -+static int dw_hdmi_detect_phy(struct dw_hdmi_qp *hdmi) -+{ -+ u8 phy_type; -+ -+ phy_type = hdmi->plat_data->phy_force_vendor ? -+ DW_HDMI_PHY_VENDOR_PHY : 0; -+ -+ if (phy_type == DW_HDMI_PHY_VENDOR_PHY) { -+ /* Vendor PHYs require support from the glue layer. */ -+ if (!hdmi->plat_data->qp_phy_ops || !hdmi->plat_data->phy_name) { -+ dev_err(hdmi->dev, -+ "Vendor HDMI PHY not supported by glue layer\n"); -+ return -ENODEV; -+ } -+ -+ hdmi->phy.ops = hdmi->plat_data->qp_phy_ops; -+ hdmi->phy.data = hdmi->plat_data->phy_data; -+ hdmi->phy.name = hdmi->plat_data->phy_name; -+ } -+ -+ return 0; -+} -+ -+void dw_hdmi_qp_cec_set_hpd(struct dw_hdmi_qp *hdmi, bool plug_in, bool change) -+{ -+ enum drm_connector_status status = plug_in ? -+ connector_status_connected : connector_status_disconnected; -+ -+ if (!plug_in) -+ cec_notifier_set_phys_addr(hdmi->cec_notifier, -+ CEC_PHYS_ADDR_INVALID); -+ -+ if (hdmi->bridge.dev) { -+ if (change && hdmi->cec_adap && -+ hdmi->cec_adap->devnode.registered) -+ cec_queue_pin_hpd_event(hdmi->cec_adap, -+ hdmi->hpd_state, -+ ktime_get()); -+ drm_bridge_hpd_notify(&hdmi->bridge, status); -+ } -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_cec_set_hpd); -+ -+static const struct regmap_config hdmi_regmap_config = { -+ .reg_bits = 32, -+ .val_bits = 32, -+ .reg_stride = 4, -+ .max_register = EARCRX_1_INT_FORCE, -+}; -+ -+static struct dw_hdmi_qp * -+__dw_hdmi_probe(struct platform_device *pdev, -+ const struct dw_hdmi_plat_data *plat_data) -+{ -+ struct device *dev = &pdev->dev; -+ struct device_node *np = dev->of_node; -+ struct device_node *ddc_node; -+ struct dw_hdmi_qp *hdmi; -+ struct dw_hdmi_qp_i2s_audio_data audio; -+ struct platform_device_info pdevinfo; -+ struct resource *iores = NULL; -+ int irq; -+ int ret; -+ -+ hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL); -+ if (!hdmi) -+ return ERR_PTR(-ENOMEM); -+ -+ hdmi->connector.stereo_allowed = 1; -+ hdmi->plat_data = plat_data; -+ hdmi->dev = dev; -+ hdmi->sample_rate = 48000; -+ hdmi->disabled = true; -+ -+ mutex_init(&hdmi->mutex); -+ mutex_init(&hdmi->audio_mutex); -+ mutex_init(&hdmi->cec_notifier_mutex); -+ spin_lock_init(&hdmi->audio_lock); -+ -+ ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0); -+ if (ddc_node) { -+ hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node); -+ of_node_put(ddc_node); -+ if (!hdmi->ddc) { -+ dev_dbg(hdmi->dev, "failed to read ddc node\n"); -+ return ERR_PTR(-EPROBE_DEFER); -+ } -+ -+ } else { -+ dev_dbg(hdmi->dev, "no ddc property found\n"); -+ } -+ -+ if (!plat_data->regm) { -+ const struct regmap_config *reg_config; -+ -+ reg_config = &hdmi_regmap_config; -+ -+ iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); -+ hdmi->regs = devm_ioremap_resource(dev, iores); -+ if (IS_ERR(hdmi->regs)) { -+ ret = PTR_ERR(hdmi->regs); -+ goto err_res; -+ } -+ -+ hdmi->regm = devm_regmap_init_mmio(dev, hdmi->regs, reg_config); -+ if (IS_ERR(hdmi->regm)) { -+ dev_err(dev, "Failed to configure regmap\n"); -+ ret = PTR_ERR(hdmi->regm); -+ goto err_res; -+ } -+ } else { -+ hdmi->regm = plat_data->regm; -+ } -+ -+ ret = dw_hdmi_detect_phy(hdmi); -+ if (ret < 0) -+ goto err_res; -+ -+ hdmi_writel(hdmi, 0, MAINUNIT_0_INT_MASK_N); -+ hdmi_writel(hdmi, 0, MAINUNIT_1_INT_MASK_N); -+ hdmi_writel(hdmi, 428571429, TIMER_BASE_CONFIG0); -+ -+ irq = platform_get_irq(pdev, 0); -+ if (irq < 0) { -+ ret = irq; -+ goto err_res; -+ } -+ -+ hdmi->avp_irq = irq; -+ ret = devm_request_threaded_irq(dev, hdmi->avp_irq, -+ dw_hdmi_qp_avp_hardirq, -+ dw_hdmi_qp_avp_irq, IRQF_SHARED, -+ dev_name(dev), hdmi); -+ if (ret) -+ goto err_res; -+ -+ irq = platform_get_irq(pdev, 2); -+ if (irq < 0) { -+ ret = irq; -+ goto err_res; -+ } -+ -+ hdmi->earc_irq = irq; -+ ret = devm_request_threaded_irq(dev, hdmi->earc_irq, -+ dw_hdmi_qp_earc_hardirq, -+ dw_hdmi_qp_earc_irq, IRQF_SHARED, -+ dev_name(dev), hdmi); -+ if (ret) -+ goto err_res; -+ -+ irq = platform_get_irq(pdev, 3); -+ if (irq < 0) { -+ ret = irq; -+ goto err_res; -+ } -+ -+ hdmi->main_irq = irq; -+ ret = devm_request_threaded_irq(dev, hdmi->main_irq, -+ dw_hdmi_qp_main_hardirq, NULL, -+ IRQF_SHARED, dev_name(dev), hdmi); -+ if (ret) -+ goto err_res; -+ -+ hdmi_init_clk_regenerator(hdmi); -+ -+ /* If DDC bus is not specified, try to register HDMI I2C bus */ -+ if (!hdmi->ddc) { -+ hdmi->ddc = dw_hdmi_i2c_adapter(hdmi); -+ if (IS_ERR(hdmi->ddc)) -+ hdmi->ddc = NULL; -+ /* -+ * Read high and low time from device tree. If not available use -+ * the default timing scl clock rate is about 99.6KHz. -+ */ -+ if (of_property_read_u32(np, "ddc-i2c-scl-high-time-ns", -+ &hdmi->i2c->scl_high_ns)) -+ hdmi->i2c->scl_high_ns = 4708; -+ if (of_property_read_u32(np, "ddc-i2c-scl-low-time-ns", -+ &hdmi->i2c->scl_low_ns)) -+ hdmi->i2c->scl_low_ns = 4916; -+ } -+ -+ hdmi->bridge.driver_private = hdmi; -+ hdmi->bridge.funcs = &dw_hdmi_bridge_funcs; -+#ifdef CONFIG_OF -+ hdmi->bridge.of_node = pdev->dev.of_node; -+#endif -+ -+ if (hdmi->phy.ops->setup_hpd) -+ hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data); -+ -+ hdmi->connector.ycbcr_420_allowed = hdmi->plat_data->ycbcr_420_allowed; -+ -+ audio.hdmi = hdmi; -+ audio.eld = hdmi->connector.eld; -+ audio.write = hdmi_writel; -+ audio.read = hdmi_readl; -+ audio.mod = hdmi_modb; -+ hdmi->enable_audio = dw_hdmi_i2s_audio_enable; -+ hdmi->disable_audio = dw_hdmi_i2s_audio_disable; -+ -+ memset(&pdevinfo, 0, sizeof(pdevinfo)); -+ pdevinfo.parent = dev; -+ pdevinfo.id = PLATFORM_DEVID_AUTO; -+ pdevinfo.name = "dw-hdmi-qp-i2s-audio"; -+ pdevinfo.data = &audio; -+ pdevinfo.size_data = sizeof(audio); -+ pdevinfo.dma_mask = DMA_BIT_MASK(32); -+ hdmi->audio = platform_device_register_full(&pdevinfo); -+ -+ hdmi->extcon = devm_extcon_dev_allocate(hdmi->dev, dw_hdmi_cable); -+ if (IS_ERR(hdmi->extcon)) { -+ dev_err(hdmi->dev, "allocate extcon failed\n"); -+ ret = PTR_ERR(hdmi->extcon); -+ goto err_res; -+ } -+ -+ ret = devm_extcon_dev_register(hdmi->dev, hdmi->extcon); -+ if (ret) { -+ dev_err(hdmi->dev, "failed to register extcon: %d\n", ret); -+ goto err_res; -+ } -+ -+ ret = extcon_set_property_capability(hdmi->extcon, EXTCON_DISP_HDMI, -+ EXTCON_PROP_DISP_HPD); -+ if (ret) { -+ dev_err(hdmi->dev, -+ "failed to set USB property capability: %d\n", ret); -+ goto err_res; -+ } -+ -+ /* Reset HDMI DDC I2C master controller and mute I2CM interrupts */ -+ if (hdmi->i2c) -+ dw_hdmi_i2c_init(hdmi); -+ -+ init_completion(&hdmi->flt_cmp); -+ init_completion(&hdmi->earc_cmp); -+ -+ if (of_property_read_bool(np, "scramble-low-rates")) -+ hdmi->scramble_low_rates = true; -+ -+ return hdmi; -+ -+err_res: -+ if (hdmi->i2c) -+ i2c_del_adapter(&hdmi->i2c->adap); -+ else -+ i2c_put_adapter(hdmi->ddc); -+ -+ return ERR_PTR(ret); -+} -+ -+static void __dw_hdmi_remove(struct dw_hdmi_qp *hdmi) -+{ -+ if (hdmi->avp_irq) -+ disable_irq(hdmi->avp_irq); -+ -+ if (hdmi->main_irq) -+ disable_irq(hdmi->main_irq); -+ -+ if (hdmi->earc_irq) -+ disable_irq(hdmi->earc_irq); -+ -+ dw_hdmi_destroy_properties(hdmi); -+ hdmi->connector.funcs->destroy(&hdmi->connector); -+ -+ if (hdmi->audio && !IS_ERR(hdmi->audio)) -+ platform_device_unregister(hdmi->audio); -+ -+ if (hdmi->bridge.encoder) -+ hdmi->bridge.encoder->funcs->destroy(hdmi->bridge.encoder); -+ -+ if (hdmi->i2c) -+ i2c_del_adapter(&hdmi->i2c->adap); -+ else -+ i2c_put_adapter(hdmi->ddc); -+} -+ -+/* ----------------------------------------------------------------------------- -+ * Bind/unbind API, used from platforms based on the component framework. -+ */ -+struct dw_hdmi_qp *dw_hdmi_qp_bind(struct platform_device *pdev, -+ struct drm_encoder *encoder, -+ struct dw_hdmi_plat_data *plat_data) -+{ -+ struct dw_hdmi_qp *hdmi; -+ int ret; -+ -+ hdmi = __dw_hdmi_probe(pdev, plat_data); -+ if (IS_ERR(hdmi)) -+ return hdmi; -+ -+ ret = drm_bridge_attach(encoder, &hdmi->bridge, NULL, 0); -+ if (ret) { -+ __dw_hdmi_remove(hdmi); -+ dev_err(hdmi->dev, "Failed to initialize bridge with drm\n"); -+ return ERR_PTR(ret); -+ } -+ -+ plat_data->connector = &hdmi->connector; -+ -+ return hdmi; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_bind); -+ -+void dw_hdmi_qp_unbind(struct dw_hdmi_qp *hdmi) -+{ -+ __dw_hdmi_remove(hdmi); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_unbind); -+ -+void dw_hdmi_qp_suspend(struct device *dev, struct dw_hdmi_qp *hdmi) -+{ -+ if (!hdmi) { -+ dev_warn(dev, "Hdmi has not been initialized\n"); -+ return; -+ } -+ -+ mutex_lock(&hdmi->mutex); -+ -+ /* -+ * When system shutdown, hdmi should be disabled. -+ * When system suspend, dw_hdmi_qp_bridge_disable will disable hdmi first. -+ * To prevent duplicate operation, we should determine whether hdmi -+ * has been disabled. -+ */ -+ if (!hdmi->disabled) -+ hdmi->disabled = true; -+ mutex_unlock(&hdmi->mutex); -+ -+ if (hdmi->avp_irq) -+ disable_irq(hdmi->avp_irq); -+ -+ if (hdmi->main_irq) -+ disable_irq(hdmi->main_irq); -+ -+ if (hdmi->earc_irq) -+ disable_irq(hdmi->earc_irq); -+ -+ pinctrl_pm_select_sleep_state(dev); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_suspend); -+ -+void dw_hdmi_qp_resume(struct device *dev, struct dw_hdmi_qp *hdmi) -+{ -+ if (!hdmi) { -+ dev_warn(dev, "Hdmi has not been initialized\n"); -+ return; -+ } -+ -+ hdmi_writel(hdmi, 0, MAINUNIT_0_INT_MASK_N); -+ hdmi_writel(hdmi, 0, MAINUNIT_1_INT_MASK_N); -+ hdmi_writel(hdmi, 428571429, TIMER_BASE_CONFIG0); -+ -+ pinctrl_pm_select_default_state(dev); -+ mutex_lock(&hdmi->mutex); -+ if (hdmi->i2c) -+ dw_hdmi_i2c_init(hdmi); -+ if (hdmi->avp_irq) -+ enable_irq(hdmi->avp_irq); -+ -+ if (hdmi->main_irq) -+ enable_irq(hdmi->main_irq); -+ -+ if (hdmi->earc_irq) -+ enable_irq(hdmi->earc_irq); -+ -+ mutex_unlock(&hdmi->mutex); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_qp_resume); -+ -+MODULE_AUTHOR("Algea Cao "); -+MODULE_DESCRIPTION("DW HDMI QP transmitter driver"); -+MODULE_LICENSE("GPL"); -+MODULE_ALIAS("platform:dw-hdmi-qp"); -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h -new file mode 100755 -index 000000000..a891c8333 ---- /dev/null -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.h -@@ -0,0 +1,824 @@ -+/* SPDX-License-Identifier: GPL-2.0 */ -+/* -+ * Copyright (C) Rockchip Electronics Co.Ltd -+ * Author: -+ * Algea Cao -+ */ -+#ifndef __DW_HDMI_QP_H__ -+#define __DW_HDMI_QP_H__ -+/* Main Unit Registers */ -+#define CORE_ID 0x0 -+#define VER_NUMBER 0x4 -+#define VER_TYPE 0x8 -+#define CONFIG_REG 0xc -+#define CONFIG_CEC BIT(28) -+#define CONFIG_AUD_UD BIT(23) -+#define CORE_TIMESTAMP_HHMM 0x14 -+#define CORE_TIMESTAMP_MMDD 0x18 -+#define CORE_TIMESTAMP_YYYY 0x1c -+/* Reset Manager Registers */ -+#define GLOBAL_SWRESET_REQUEST 0x40 -+#define EARCRX_CMDC_SWINIT_P BIT(27) -+#define AVP_DATAPATH_PACKET_AUDIO_SWINIT_P BIT(10) -+#define GLOBAL_SWDISABLE 0x44 -+#define CEC_SWDISABLE BIT(17) -+#define AVP_DATAPATH_PACKET_AUDIO_SWDISABLE BIT(10) -+#define AVP_DATAPATH_VIDEO_SWDISABLE BIT(6) -+#define RESET_MANAGER_CONFIG0 0x48 -+#define RESET_MANAGER_STATUS0 0x50 -+#define RESET_MANAGER_STATUS1 0x54 -+#define RESET_MANAGER_STATUS2 0x58 -+/* Timer Base Registers */ -+#define TIMER_BASE_CONFIG0 0x80 -+#define TIMER_BASE_STATUS0 0x84 -+/* CMU Registers */ -+#define CMU_CONFIG0 0xa0 -+#define CMU_CONFIG1 0xa4 -+#define CMU_CONFIG2 0xa8 -+#define CMU_CONFIG3 0xac -+#define CMU_STATUS 0xb0 -+#define EARC_BPCLK_OFF BIT(9) -+#define AUDCLK_OFF BIT(7) -+#define LINKQPCLK_OFF BIT(5) -+#define VIDQPCLK_OFF BIT(3) -+#define IPI_CLK_OFF BIT(1) -+#define CMU_IPI_CLK_FREQ 0xb4 -+#define CMU_VIDQPCLK_FREQ 0xb8 -+#define CMU_LINKQPCLK_FREQ 0xbc -+#define CMU_AUDQPCLK_FREQ 0xc0 -+#define CMU_EARC_BPCLK_FREQ 0xc4 -+/* I2CM Registers */ -+#define I2CM_SM_SCL_CONFIG0 0xe0 -+#define I2CM_FM_SCL_CONFIG0 0xe4 -+#define I2CM_CONFIG0 0xe8 -+#define I2CM_CONTROL0 0xec -+#define I2CM_STATUS0 0xf0 -+#define I2CM_INTERFACE_CONTROL0 0xf4 -+#define I2CM_ADDR 0xff000 -+#define I2CM_SLVADDR 0xfe0 -+#define I2CM_WR_MASK 0x1e -+#define I2CM_EXT_READ BIT(4) -+#define I2CM_SHORT_READ BIT(3) -+#define I2CM_FM_READ BIT(2) -+#define I2CM_FM_WRITE BIT(1) -+#define I2CM_FM_EN BIT(0) -+#define I2CM_INTERFACE_CONTROL1 0xf8 -+#define I2CM_SEG_PTR 0x7f80 -+#define I2CM_SEG_ADDR 0x7f -+#define I2CM_INTERFACE_WRDATA_0_3 0xfc -+#define I2CM_INTERFACE_WRDATA_4_7 0x100 -+#define I2CM_INTERFACE_WRDATA_8_11 0x104 -+#define I2CM_INTERFACE_WRDATA_12_15 0x108 -+#define I2CM_INTERFACE_RDDATA_0_3 0x10c -+#define I2CM_INTERFACE_RDDATA_4_7 0x110 -+#define I2CM_INTERFACE_RDDATA_8_11 0x114 -+#define I2CM_INTERFACE_RDDATA_12_15 0x118 -+/* SCDC Registers */ -+#define SCDC_CONFIG0 0x140 -+#define SCDC_I2C_FM_EN BIT(12) -+#define SCDC_UPD_FLAGS_AUTO_CLR BIT(6) -+#define SCDC_UPD_FLAGS_POLL_EN BIT(4) -+#define SCDC_CONTROL0 0x148 -+#define SCDC_STATUS0 0x150 -+#define STATUS_UPDATE BIT(0) -+#define FRL_START BIT(4) -+#define FLT_UPDATE BIT(5) -+/* FLT Registers */ -+#define FLT_CONFIG0 0x160 -+#define FLT_CONFIG1 0x164 -+#define FLT_CONFIG2 0x168 -+#define FLT_CONTROL0 0x170 -+/* Main Unit 2 Registers */ -+#define MAINUNIT_STATUS0 0x180 -+/* Video Interface Registers */ -+#define VIDEO_INTERFACE_CONFIG0 0x800 -+#define VIDEO_INTERFACE_CONFIG1 0x804 -+#define VIDEO_INTERFACE_CONFIG2 0x808 -+#define VIDEO_INTERFACE_CONTROL0 0x80c -+#define VIDEO_INTERFACE_STATUS0 0x814 -+/* Video Packing Registers */ -+#define VIDEO_PACKING_CONFIG0 0x81c -+/* Audio Interface Registers */ -+#define AUDIO_INTERFACE_CONFIG0 0x820 -+#define AUD_IF_SEL_MSK 0x3 -+#define AUD_IF_SPDIF 0x2 -+#define AUD_IF_I2S 0x1 -+#define AUD_IF_PAI 0x0 -+#define AUD_FIFO_INIT_ON_OVF_MSK BIT(2) -+#define AUD_FIFO_INIT_ON_OVF_EN BIT(2) -+#define I2S_LINES_EN_MSK GENMASK(7, 4) -+#define I2S_LINES_EN(x) BIT(x + 4) -+#define I2S_BPCUV_RCV_MSK BIT(12) -+#define I2S_BPCUV_RCV_EN BIT(12) -+#define I2S_BPCUV_RCV_DIS 0 -+#define SPDIF_LINES_EN GENMASK(19, 16) -+#define AUD_FORMAT_MSK GENMASK(26, 24) -+#define AUD_3DOBA (0x7 << 24) -+#define AUD_3DASP (0x6 << 24) -+#define AUD_MSOBA (0x5 << 24) -+#define AUD_MSASP (0x4 << 24) -+#define AUD_HBR (0x3 << 24) -+#define AUD_DST (0x2 << 24) -+#define AUD_OBA (0x1 << 24) -+#define AUD_ASP (0x0 << 24) -+#define AUDIO_INTERFACE_CONFIG1 0x824 -+#define AUDIO_INTERFACE_CONTROL0 0x82c -+#define AUDIO_FIFO_CLR_P BIT(0) -+#define AUDIO_INTERFACE_STATUS0 0x834 -+/* Frame Composer Registers */ -+#define FRAME_COMPOSER_CONFIG0 0x840 -+#define FRAME_COMPOSER_CONFIG1 0x844 -+#define FRAME_COMPOSER_CONFIG2 0x848 -+#define FRAME_COMPOSER_CONFIG3 0x84c -+#define FRAME_COMPOSER_CONFIG4 0x850 -+#define FRAME_COMPOSER_CONFIG5 0x854 -+#define FRAME_COMPOSER_CONFIG6 0x858 -+#define FRAME_COMPOSER_CONFIG7 0x85c -+#define FRAME_COMPOSER_CONFIG8 0x860 -+#define FRAME_COMPOSER_CONFIG9 0x864 -+#define FRAME_COMPOSER_CONTROL0 0x86c -+/* Video Monitor Registers */ -+#define VIDEO_MONITOR_CONFIG0 0x880 -+#define VIDEO_MONITOR_STATUS0 0x884 -+#define VIDEO_MONITOR_STATUS1 0x888 -+#define VIDEO_MONITOR_STATUS2 0x88c -+#define VIDEO_MONITOR_STATUS3 0x890 -+#define VIDEO_MONITOR_STATUS4 0x894 -+#define VIDEO_MONITOR_STATUS5 0x898 -+#define VIDEO_MONITOR_STATUS6 0x89c -+/* HDCP2 Logic Registers */ -+#define HDCP2LOGIC_CONFIG0 0x8e0 -+#define HDCP2_BYPASS BIT(0) -+#define HDCP2LOGIC_ESM_GPIO_IN 0x8e4 -+#define HDCP2LOGIC_ESM_GPIO_OUT 0x8e8 -+/* HDCP14 Registers */ -+#define HDCP14_CONFIG0 0x900 -+#define HDCP14_CONFIG1 0x904 -+#define HDCP14_CONFIG2 0x908 -+#define HDCP14_CONFIG3 0x90c -+#define HDCP14_KEY_SEED 0x914 -+#define HDCP14_KEY_H 0x918 -+#define HDCP14_KEY_L 0x91c -+#define HDCP14_KEY_STATUS 0x920 -+#define HDCP14_AKSV_H 0x924 -+#define HDCP14_AKSV_L 0x928 -+#define HDCP14_AN_H 0x92c -+#define HDCP14_AN_L 0x930 -+#define HDCP14_STATUS0 0x934 -+#define HDCP14_STATUS1 0x938 -+/* Scrambler Registers */ -+#define SCRAMB_CONFIG0 0x960 -+/* Video Configuration Registers */ -+#define LINK_CONFIG0 0x968 -+#define OPMODE_FRL_4LANES BIT(8) -+#define OPMODE_DVI BIT(4) -+#define OPMODE_FRL BIT(0) -+/* TMDS FIFO Registers */ -+#define TMDS_FIFO_CONFIG0 0x970 -+#define TMDS_FIFO_CONTROL0 0x974 -+/* FRL RSFEC Registers */ -+#define FRL_RSFEC_CONFIG0 0xa20 -+#define FRL_RSFEC_STATUS0 0xa30 -+/* FRL Packetizer Registers */ -+#define FRL_PKTZ_CONFIG0 0xa40 -+#define FRL_PKTZ_CONTROL0 0xa44 -+#define FRL_PKTZ_CONTROL1 0xa50 -+#define FRL_PKTZ_STATUS1 0xa54 -+/* Packet Scheduler Registers */ -+#define PKTSCHED_CONFIG0 0xa80 -+#define PKTSCHED_PRQUEUE0_CONFIG0 0xa84 -+#define PKTSCHED_PRQUEUE1_CONFIG0 0xa88 -+#define PKTSCHED_PRQUEUE2_CONFIG0 0xa8c -+#define PKTSCHED_PRQUEUE2_CONFIG1 0xa90 -+#define PKTSCHED_PRQUEUE2_CONFIG2 0xa94 -+#define PKTSCHED_PKT_CONFIG0 0xa98 -+#define PKTSCHED_PKT_CONFIG1 0xa9c -+#define PKTSCHED_PKT_CONFIG2 0xaa0 -+#define PKTSCHED_PKT_CONFIG3 0xaa4 -+#define PKTSCHED_PKT_EN 0xaa8 -+#define PKTSCHED_DRMI_TX_EN BIT(17) -+#define PKTSCHED_AUDI_TX_EN BIT(15) -+#define PKTSCHED_AVI_TX_EN BIT(13) -+#define PKTSCHED_EMP_CVTEM_TX_EN BIT(10) -+#define PKTSCHED_AMD_TX_EN BIT(8) -+#define PKTSCHED_GCP_TX_EN BIT(3) -+#define PKTSCHED_AUDS_TX_EN BIT(2) -+#define PKTSCHED_ACR_TX_EN BIT(1) -+#define PKTSCHED_PKT_CONTROL0 0xaac -+#define PKTSCHED_PKT_SEND 0xab0 -+#define PKTSCHED_PKT_STATUS0 0xab4 -+#define PKTSCHED_PKT_STATUS1 0xab8 -+#define PKT_NULL_CONTENTS0 0xb00 -+#define PKT_NULL_CONTENTS1 0xb04 -+#define PKT_NULL_CONTENTS2 0xb08 -+#define PKT_NULL_CONTENTS3 0xb0c -+#define PKT_NULL_CONTENTS4 0xb10 -+#define PKT_NULL_CONTENTS5 0xb14 -+#define PKT_NULL_CONTENTS6 0xb18 -+#define PKT_NULL_CONTENTS7 0xb1c -+#define PKT_ACP_CONTENTS0 0xb20 -+#define PKT_ACP_CONTENTS1 0xb24 -+#define PKT_ACP_CONTENTS2 0xb28 -+#define PKT_ACP_CONTENTS3 0xb2c -+#define PKT_ACP_CONTENTS4 0xb30 -+#define PKT_ACP_CONTENTS5 0xb34 -+#define PKT_ACP_CONTENTS6 0xb38 -+#define PKT_ACP_CONTENTS7 0xb3c -+#define PKT_ISRC1_CONTENTS0 0xb40 -+#define PKT_ISRC1_CONTENTS1 0xb44 -+#define PKT_ISRC1_CONTENTS2 0xb48 -+#define PKT_ISRC1_CONTENTS3 0xb4c -+#define PKT_ISRC1_CONTENTS4 0xb50 -+#define PKT_ISRC1_CONTENTS5 0xb54 -+#define PKT_ISRC1_CONTENTS6 0xb58 -+#define PKT_ISRC1_CONTENTS7 0xb5c -+#define PKT_ISRC2_CONTENTS0 0xb60 -+#define PKT_ISRC2_CONTENTS1 0xb64 -+#define PKT_ISRC2_CONTENTS2 0xb68 -+#define PKT_ISRC2_CONTENTS3 0xb6c -+#define PKT_ISRC2_CONTENTS4 0xb70 -+#define PKT_ISRC2_CONTENTS5 0xb74 -+#define PKT_ISRC2_CONTENTS6 0xb78 -+#define PKT_ISRC2_CONTENTS7 0xb7c -+#define PKT_GMD_CONTENTS0 0xb80 -+#define PKT_GMD_CONTENTS1 0xb84 -+#define PKT_GMD_CONTENTS2 0xb88 -+#define PKT_GMD_CONTENTS3 0xb8c -+#define PKT_GMD_CONTENTS4 0xb90 -+#define PKT_GMD_CONTENTS5 0xb94 -+#define PKT_GMD_CONTENTS6 0xb98 -+#define PKT_GMD_CONTENTS7 0xb9c -+#define PKT_AMD_CONTENTS0 0xba0 -+#define PKT_AMD_CONTENTS1 0xba4 -+#define PKT_AMD_CONTENTS2 0xba8 -+#define PKT_AMD_CONTENTS3 0xbac -+#define PKT_AMD_CONTENTS4 0xbb0 -+#define PKT_AMD_CONTENTS5 0xbb4 -+#define PKT_AMD_CONTENTS6 0xbb8 -+#define PKT_AMD_CONTENTS7 0xbbc -+#define PKT_VSI_CONTENTS0 0xbc0 -+#define PKT_VSI_CONTENTS1 0xbc4 -+#define PKT_VSI_CONTENTS2 0xbc8 -+#define PKT_VSI_CONTENTS3 0xbcc -+#define PKT_VSI_CONTENTS4 0xbd0 -+#define PKT_VSI_CONTENTS5 0xbd4 -+#define PKT_VSI_CONTENTS6 0xbd8 -+#define PKT_VSI_CONTENTS7 0xbdc -+#define PKT_AVI_CONTENTS0 0xbe0 -+#define HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT BIT(4) -+#define HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR 0x04 -+#define HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR 0x08 -+#define HDMI_FC_AVICONF2_IT_CONTENT_VALID 0x80 -+#define PKT_AVI_CONTENTS1 0xbe4 -+#define PKT_AVI_CONTENTS2 0xbe8 -+#define PKT_AVI_CONTENTS3 0xbec -+#define PKT_AVI_CONTENTS4 0xbf0 -+#define PKT_AVI_CONTENTS5 0xbf4 -+#define PKT_AVI_CONTENTS6 0xbf8 -+#define PKT_AVI_CONTENTS7 0xbfc -+#define PKT_SPDI_CONTENTS0 0xc00 -+#define PKT_SPDI_CONTENTS1 0xc04 -+#define PKT_SPDI_CONTENTS2 0xc08 -+#define PKT_SPDI_CONTENTS3 0xc0c -+#define PKT_SPDI_CONTENTS4 0xc10 -+#define PKT_SPDI_CONTENTS5 0xc14 -+#define PKT_SPDI_CONTENTS6 0xc18 -+#define PKT_SPDI_CONTENTS7 0xc1c -+#define PKT_AUDI_CONTENTS0 0xc20 -+#define PKT_AUDI_CONTENTS1 0xc24 -+#define PKT_AUDI_CONTENTS2 0xc28 -+#define PKT_AUDI_CONTENTS3 0xc2c -+#define PKT_AUDI_CONTENTS4 0xc30 -+#define PKT_AUDI_CONTENTS5 0xc34 -+#define PKT_AUDI_CONTENTS6 0xc38 -+#define PKT_AUDI_CONTENTS7 0xc3c -+#define PKT_NVI_CONTENTS0 0xc40 -+#define PKT_NVI_CONTENTS1 0xc44 -+#define PKT_NVI_CONTENTS2 0xc48 -+#define PKT_NVI_CONTENTS3 0xc4c -+#define PKT_NVI_CONTENTS4 0xc50 -+#define PKT_NVI_CONTENTS5 0xc54 -+#define PKT_NVI_CONTENTS6 0xc58 -+#define PKT_NVI_CONTENTS7 0xc5c -+#define PKT_DRMI_CONTENTS0 0xc60 -+#define PKT_DRMI_CONTENTS1 0xc64 -+#define PKT_DRMI_CONTENTS2 0xc68 -+#define PKT_DRMI_CONTENTS3 0xc6c -+#define PKT_DRMI_CONTENTS4 0xc70 -+#define PKT_DRMI_CONTENTS5 0xc74 -+#define PKT_DRMI_CONTENTS6 0xc78 -+#define PKT_DRMI_CONTENTS7 0xc7c -+#define PKT_GHDMI1_CONTENTS0 0xc80 -+#define PKT_GHDMI1_CONTENTS1 0xc84 -+#define PKT_GHDMI1_CONTENTS2 0xc88 -+#define PKT_GHDMI1_CONTENTS3 0xc8c -+#define PKT_GHDMI1_CONTENTS4 0xc90 -+#define PKT_GHDMI1_CONTENTS5 0xc94 -+#define PKT_GHDMI1_CONTENTS6 0xc98 -+#define PKT_GHDMI1_CONTENTS7 0xc9c -+#define PKT_GHDMI2_CONTENTS0 0xca0 -+#define PKT_GHDMI2_CONTENTS1 0xca4 -+#define PKT_GHDMI2_CONTENTS2 0xca8 -+#define PKT_GHDMI2_CONTENTS3 0xcac -+#define PKT_GHDMI2_CONTENTS4 0xcb0 -+#define PKT_GHDMI2_CONTENTS5 0xcb4 -+#define PKT_GHDMI2_CONTENTS6 0xcb8 -+#define PKT_GHDMI2_CONTENTS7 0xcbc -+/* EMP Packetizer Registers */ -+#define PKT_EMP_CONFIG0 0xce0 -+#define PKT_EMP_CONTROL0 0xcec -+#define PKT_EMP_CONTROL1 0xcf0 -+#define PKT_EMP_CONTROL2 0xcf4 -+#define PKT_EMP_VTEM_CONTENTS0 0xd00 -+#define PKT_EMP_VTEM_CONTENTS1 0xd04 -+#define PKT_EMP_VTEM_CONTENTS2 0xd08 -+#define PKT_EMP_VTEM_CONTENTS3 0xd0c -+#define PKT_EMP_VTEM_CONTENTS4 0xd10 -+#define PKT_EMP_VTEM_CONTENTS5 0xd14 -+#define PKT_EMP_VTEM_CONTENTS6 0xd18 -+#define PKT_EMP_VTEM_CONTENTS7 0xd1c -+#define PKT0_EMP_CVTEM_CONTENTS0 0xd20 -+#define PKT0_EMP_CVTEM_CONTENTS1 0xd24 -+#define PKT0_EMP_CVTEM_CONTENTS2 0xd28 -+#define PKT0_EMP_CVTEM_CONTENTS3 0xd2c -+#define PKT0_EMP_CVTEM_CONTENTS4 0xd30 -+#define PKT0_EMP_CVTEM_CONTENTS5 0xd34 -+#define PKT0_EMP_CVTEM_CONTENTS6 0xd38 -+#define PKT0_EMP_CVTEM_CONTENTS7 0xd3c -+#define PKT1_EMP_CVTEM_CONTENTS0 0xd40 -+#define PKT1_EMP_CVTEM_CONTENTS1 0xd44 -+#define PKT1_EMP_CVTEM_CONTENTS2 0xd48 -+#define PKT1_EMP_CVTEM_CONTENTS3 0xd4c -+#define PKT1_EMP_CVTEM_CONTENTS4 0xd50 -+#define PKT1_EMP_CVTEM_CONTENTS5 0xd54 -+#define PKT1_EMP_CVTEM_CONTENTS6 0xd58 -+#define PKT1_EMP_CVTEM_CONTENTS7 0xd5c -+#define PKT2_EMP_CVTEM_CONTENTS0 0xd60 -+#define PKT2_EMP_CVTEM_CONTENTS1 0xd64 -+#define PKT2_EMP_CVTEM_CONTENTS2 0xd68 -+#define PKT2_EMP_CVTEM_CONTENTS3 0xd6c -+#define PKT2_EMP_CVTEM_CONTENTS4 0xd70 -+#define PKT2_EMP_CVTEM_CONTENTS5 0xd74 -+#define PKT2_EMP_CVTEM_CONTENTS6 0xd78 -+#define PKT2_EMP_CVTEM_CONTENTS7 0xd7c -+#define PKT3_EMP_CVTEM_CONTENTS0 0xd80 -+#define PKT3_EMP_CVTEM_CONTENTS1 0xd84 -+#define PKT3_EMP_CVTEM_CONTENTS2 0xd88 -+#define PKT3_EMP_CVTEM_CONTENTS3 0xd8c -+#define PKT3_EMP_CVTEM_CONTENTS4 0xd90 -+#define PKT3_EMP_CVTEM_CONTENTS5 0xd94 -+#define PKT3_EMP_CVTEM_CONTENTS6 0xd98 -+#define PKT3_EMP_CVTEM_CONTENTS7 0xd9c -+#define PKT4_EMP_CVTEM_CONTENTS0 0xda0 -+#define PKT4_EMP_CVTEM_CONTENTS1 0xda4 -+#define PKT4_EMP_CVTEM_CONTENTS2 0xda8 -+#define PKT4_EMP_CVTEM_CONTENTS3 0xdac -+#define PKT4_EMP_CVTEM_CONTENTS4 0xdb0 -+#define PKT4_EMP_CVTEM_CONTENTS5 0xdb4 -+#define PKT4_EMP_CVTEM_CONTENTS6 0xdb8 -+#define PKT4_EMP_CVTEM_CONTENTS7 0xdbc -+#define PKT5_EMP_CVTEM_CONTENTS0 0xdc0 -+#define PKT5_EMP_CVTEM_CONTENTS1 0xdc4 -+#define PKT5_EMP_CVTEM_CONTENTS2 0xdc8 -+#define PKT5_EMP_CVTEM_CONTENTS3 0xdcc -+#define PKT5_EMP_CVTEM_CONTENTS4 0xdd0 -+#define PKT5_EMP_CVTEM_CONTENTS5 0xdd4 -+#define PKT5_EMP_CVTEM_CONTENTS6 0xdd8 -+#define PKT5_EMP_CVTEM_CONTENTS7 0xddc -+/* Audio Packetizer Registers */ -+#define AUDPKT_CONTROL0 0xe20 -+#define AUDPKT_CHSTATUS_OVR_EN_MASK BIT(0) -+#define AUDPKT_CHSTATUS_OVR_EN BIT(0) -+#define AUDPKT_CONTROL1 0xe24 -+#define AUDPKT_ACR_CONTROL0 0xe40 -+#define AUDPKT_ACR_N_VALUE 0xfffff -+#define AUDPKT_ACR_CONTROL1 0xe44 -+#define AUDPKT_ACR_CTS_OVR_VAL_MSK GENMASK(23, 4) -+#define AUDPKT_ACR_CTS_OVR_VAL(x) ((x) << 4) -+#define AUDPKT_ACR_CTS_OVR_EN_MSK BIT(1) -+#define AUDPKT_ACR_CTS_OVR_EN BIT(1) -+#define AUDPKT_ACR_STATUS0 0xe4c -+#define AUDPKT_CHSTATUS_OVR0 0xe60 -+#define AUDPKT_CHSTATUS_OVR1 0xe64 -+/* IEC60958 Byte 3: Sampleing frenuency Bits 24 to 27 */ -+#define AUDPKT_CHSTATUS_SR_MASK GENMASK(3, 0) -+#define AUDPKT_CHSTATUS_SR_22050 0x4 -+#define AUDPKT_CHSTATUS_SR_24000 0x6 -+#define AUDPKT_CHSTATUS_SR_32000 0x3 -+#define AUDPKT_CHSTATUS_SR_44100 0x0 -+#define AUDPKT_CHSTATUS_SR_48000 0x2 -+#define AUDPKT_CHSTATUS_SR_88200 0x8 -+#define AUDPKT_CHSTATUS_SR_96000 0xa -+#define AUDPKT_CHSTATUS_SR_176400 0xc -+#define AUDPKT_CHSTATUS_SR_192000 0xe -+#define AUDPKT_CHSTATUS_SR_768000 0x9 -+#define AUDPKT_CHSTATUS_SR_NOT_INDICATED 0x1 -+/* IEC60958 Byte 4: Original Sampleing frenuency Bits 36 to 39 */ -+#define AUDPKT_CHSTATUS_0SR_MASK GENMASK(15, 12) -+#define AUDPKT_CHSTATUS_OSR_8000 0x6 -+#define AUDPKT_CHSTATUS_OSR_11025 0xa -+#define AUDPKT_CHSTATUS_OSR_12000 0x2 -+#define AUDPKT_CHSTATUS_OSR_16000 0x8 -+#define AUDPKT_CHSTATUS_OSR_22050 0xb -+#define AUDPKT_CHSTATUS_OSR_24000 0x9 -+#define AUDPKT_CHSTATUS_OSR_32000 0xc -+#define AUDPKT_CHSTATUS_OSR_44100 0xf -+#define AUDPKT_CHSTATUS_OSR_48000 0xd -+#define AUDPKT_CHSTATUS_OSR_88200 0x7 -+#define AUDPKT_CHSTATUS_OSR_96000 0x5 -+#define AUDPKT_CHSTATUS_OSR_176400 0x3 -+#define AUDPKT_CHSTATUS_OSR_192000 0x1 -+#define AUDPKT_CHSTATUS_OSR_NOT_INDICATED 0x0 -+#define AUDPKT_CHSTATUS_OVR2 0xe68 -+#define AUDPKT_CHSTATUS_OVR3 0xe6c -+#define AUDPKT_CHSTATUS_OVR4 0xe70 -+#define AUDPKT_CHSTATUS_OVR5 0xe74 -+#define AUDPKT_CHSTATUS_OVR6 0xe78 -+#define AUDPKT_CHSTATUS_OVR7 0xe7c -+#define AUDPKT_CHSTATUS_OVR8 0xe80 -+#define AUDPKT_CHSTATUS_OVR9 0xe84 -+#define AUDPKT_CHSTATUS_OVR10 0xe88 -+#define AUDPKT_CHSTATUS_OVR11 0xe8c -+#define AUDPKT_CHSTATUS_OVR12 0xe90 -+#define AUDPKT_CHSTATUS_OVR13 0xe94 -+#define AUDPKT_CHSTATUS_OVR14 0xe98 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC0 0xea0 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC1 0xea4 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC2 0xea8 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC3 0xeac -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC4 0xeb0 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC5 0xeb4 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC6 0xeb8 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC7 0xebc -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC8 0xec0 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC9 0xec4 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC10 0xec8 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC11 0xecc -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC12 0xed0 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC13 0xed4 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC14 0xed8 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC15 0xedc -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC16 0xee0 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC17 0xee4 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC18 0xee8 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC19 0xeec -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC20 0xef0 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC21 0xef4 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC22 0xef8 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC23 0xefc -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC24 0xf00 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC25 0xf04 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC26 0xf08 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC27 0xf0c -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC28 0xf10 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC29 0xf14 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC30 0xf18 -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC31 0xf1c -+#define AUDPKT_USRDATA_OVR_MSG_GENERIC32 0xf20 -+#define AUDPKT_VBIT_OVR0 0xf24 -+/* CEC Registers */ -+#define CEC_TX_CONTROL 0x1000 -+#define CEC_STATUS 0x1004 -+#define CEC_CONFIG 0x1008 -+#define CEC_ADDR 0x100c -+#define CEC_TX_COUNT 0x1020 -+#define CEC_TX_DATA3_0 0x1024 -+#define CEC_TX_DATA7_4 0x1028 -+#define CEC_TX_DATA11_8 0x102c -+#define CEC_TX_DATA15_12 0x1030 -+#define CEC_RX_COUNT_STATUS 0x1040 -+#define CEC_RX_DATA3_0 0x1044 -+#define CEC_RX_DATA7_4 0x1048 -+#define CEC_RX_DATA11_8 0x104c -+#define CEC_RX_DATA15_12 0x1050 -+#define CEC_LOCK_CONTROL 0x1054 -+#define CEC_RXQUAL_BITTIME_CONFIG 0x1060 -+#define CEC_RX_BITTIME_CONFIG 0x1064 -+#define CEC_TX_BITTIME_CONFIG 0x1068 -+/* eARC RX CMDC Registers */ -+#define EARCRX_CMDC_CONFIG0 0x1800 -+#define EARCRX_XACTREAD_STOP_CFG BIT(26) -+#define EARCRX_XACTREAD_RETRY_CFG BIT(25) -+#define EARCRX_CMDC_DSCVR_EARCVALID0_TO_DISC1 BIT(24) -+#define EARCRX_CMDC_XACT_RESTART_EN BIT(18) -+#define EARCRX_CMDC_CONFIG1 0x1804 -+#define EARCRX_CMDC_CONTROL 0x1808 -+#define EARCRX_CMDC_HEARTBEAT_LOSS_EN BIT(4) -+#define EARCRX_CMDC_DISCOVERY_EN BIT(3) -+#define EARCRX_CONNECTOR_HPD BIT(1) -+#define EARCRX_CMDC_WHITELIST0_CONFIG 0x180c -+#define EARCRX_CMDC_WHITELIST1_CONFIG 0x1810 -+#define EARCRX_CMDC_WHITELIST2_CONFIG 0x1814 -+#define EARCRX_CMDC_WHITELIST3_CONFIG 0x1818 -+#define EARCRX_CMDC_STATUS 0x181c -+#define EARCRX_CMDC_XACT_INFO 0x1820 -+#define EARCRX_CMDC_XACT_ACTION 0x1824 -+#define EARCRX_CMDC_HEARTBEAT_RXSTAT_SE 0x1828 -+#define EARCRX_CMDC_HEARTBEAT_STATUS 0x182c -+#define EARCRX_CMDC_XACT_WR0 0x1840 -+#define EARCRX_CMDC_XACT_WR1 0x1844 -+#define EARCRX_CMDC_XACT_WR2 0x1848 -+#define EARCRX_CMDC_XACT_WR3 0x184c -+#define EARCRX_CMDC_XACT_WR4 0x1850 -+#define EARCRX_CMDC_XACT_WR5 0x1854 -+#define EARCRX_CMDC_XACT_WR6 0x1858 -+#define EARCRX_CMDC_XACT_WR7 0x185c -+#define EARCRX_CMDC_XACT_WR8 0x1860 -+#define EARCRX_CMDC_XACT_WR9 0x1864 -+#define EARCRX_CMDC_XACT_WR10 0x1868 -+#define EARCRX_CMDC_XACT_WR11 0x186c -+#define EARCRX_CMDC_XACT_WR12 0x1870 -+#define EARCRX_CMDC_XACT_WR13 0x1874 -+#define EARCRX_CMDC_XACT_WR14 0x1878 -+#define EARCRX_CMDC_XACT_WR15 0x187c -+#define EARCRX_CMDC_XACT_WR16 0x1880 -+#define EARCRX_CMDC_XACT_WR17 0x1884 -+#define EARCRX_CMDC_XACT_WR18 0x1888 -+#define EARCRX_CMDC_XACT_WR19 0x188c -+#define EARCRX_CMDC_XACT_WR20 0x1890 -+#define EARCRX_CMDC_XACT_WR21 0x1894 -+#define EARCRX_CMDC_XACT_WR22 0x1898 -+#define EARCRX_CMDC_XACT_WR23 0x189c -+#define EARCRX_CMDC_XACT_WR24 0x18a0 -+#define EARCRX_CMDC_XACT_WR25 0x18a4 -+#define EARCRX_CMDC_XACT_WR26 0x18a8 -+#define EARCRX_CMDC_XACT_WR27 0x18ac -+#define EARCRX_CMDC_XACT_WR28 0x18b0 -+#define EARCRX_CMDC_XACT_WR29 0x18b4 -+#define EARCRX_CMDC_XACT_WR30 0x18b8 -+#define EARCRX_CMDC_XACT_WR31 0x18bc -+#define EARCRX_CMDC_XACT_WR32 0x18c0 -+#define EARCRX_CMDC_XACT_WR33 0x18c4 -+#define EARCRX_CMDC_XACT_WR34 0x18c8 -+#define EARCRX_CMDC_XACT_WR35 0x18cc -+#define EARCRX_CMDC_XACT_WR36 0x18d0 -+#define EARCRX_CMDC_XACT_WR37 0x18d4 -+#define EARCRX_CMDC_XACT_WR38 0x18d8 -+#define EARCRX_CMDC_XACT_WR39 0x18dc -+#define EARCRX_CMDC_XACT_WR40 0x18e0 -+#define EARCRX_CMDC_XACT_WR41 0x18e4 -+#define EARCRX_CMDC_XACT_WR42 0x18e8 -+#define EARCRX_CMDC_XACT_WR43 0x18ec -+#define EARCRX_CMDC_XACT_WR44 0x18f0 -+#define EARCRX_CMDC_XACT_WR45 0x18f4 -+#define EARCRX_CMDC_XACT_WR46 0x18f8 -+#define EARCRX_CMDC_XACT_WR47 0x18fc -+#define EARCRX_CMDC_XACT_WR48 0x1900 -+#define EARCRX_CMDC_XACT_WR49 0x1904 -+#define EARCRX_CMDC_XACT_WR50 0x1908 -+#define EARCRX_CMDC_XACT_WR51 0x190c -+#define EARCRX_CMDC_XACT_WR52 0x1910 -+#define EARCRX_CMDC_XACT_WR53 0x1914 -+#define EARCRX_CMDC_XACT_WR54 0x1918 -+#define EARCRX_CMDC_XACT_WR55 0x191c -+#define EARCRX_CMDC_XACT_WR56 0x1920 -+#define EARCRX_CMDC_XACT_WR57 0x1924 -+#define EARCRX_CMDC_XACT_WR58 0x1928 -+#define EARCRX_CMDC_XACT_WR59 0x192c -+#define EARCRX_CMDC_XACT_WR60 0x1930 -+#define EARCRX_CMDC_XACT_WR61 0x1934 -+#define EARCRX_CMDC_XACT_WR62 0x1938 -+#define EARCRX_CMDC_XACT_WR63 0x193c -+#define EARCRX_CMDC_XACT_WR64 0x1940 -+#define EARCRX_CMDC_XACT_RD0 0x1960 -+#define EARCRX_CMDC_XACT_RD1 0x1964 -+#define EARCRX_CMDC_XACT_RD2 0x1968 -+#define EARCRX_CMDC_XACT_RD3 0x196c -+#define EARCRX_CMDC_XACT_RD4 0x1970 -+#define EARCRX_CMDC_XACT_RD5 0x1974 -+#define EARCRX_CMDC_XACT_RD6 0x1978 -+#define EARCRX_CMDC_XACT_RD7 0x197c -+#define EARCRX_CMDC_XACT_RD8 0x1980 -+#define EARCRX_CMDC_XACT_RD9 0x1984 -+#define EARCRX_CMDC_XACT_RD10 0x1988 -+#define EARCRX_CMDC_XACT_RD11 0x198c -+#define EARCRX_CMDC_XACT_RD12 0x1990 -+#define EARCRX_CMDC_XACT_RD13 0x1994 -+#define EARCRX_CMDC_XACT_RD14 0x1998 -+#define EARCRX_CMDC_XACT_RD15 0x199c -+#define EARCRX_CMDC_XACT_RD16 0x19a0 -+#define EARCRX_CMDC_XACT_RD17 0x19a4 -+#define EARCRX_CMDC_XACT_RD18 0x19a8 -+#define EARCRX_CMDC_XACT_RD19 0x19ac -+#define EARCRX_CMDC_XACT_RD20 0x19b0 -+#define EARCRX_CMDC_XACT_RD21 0x19b4 -+#define EARCRX_CMDC_XACT_RD22 0x19b8 -+#define EARCRX_CMDC_XACT_RD23 0x19bc -+#define EARCRX_CMDC_XACT_RD24 0x19c0 -+#define EARCRX_CMDC_XACT_RD25 0x19c4 -+#define EARCRX_CMDC_XACT_RD26 0x19c8 -+#define EARCRX_CMDC_XACT_RD27 0x19cc -+#define EARCRX_CMDC_XACT_RD28 0x19d0 -+#define EARCRX_CMDC_XACT_RD29 0x19d4 -+#define EARCRX_CMDC_XACT_RD30 0x19d8 -+#define EARCRX_CMDC_XACT_RD31 0x19dc -+#define EARCRX_CMDC_XACT_RD32 0x19e0 -+#define EARCRX_CMDC_XACT_RD33 0x19e4 -+#define EARCRX_CMDC_XACT_RD34 0x19e8 -+#define EARCRX_CMDC_XACT_RD35 0x19ec -+#define EARCRX_CMDC_XACT_RD36 0x19f0 -+#define EARCRX_CMDC_XACT_RD37 0x19f4 -+#define EARCRX_CMDC_XACT_RD38 0x19f8 -+#define EARCRX_CMDC_XACT_RD39 0x19fc -+#define EARCRX_CMDC_XACT_RD40 0x1a00 -+#define EARCRX_CMDC_XACT_RD41 0x1a04 -+#define EARCRX_CMDC_XACT_RD42 0x1a08 -+#define EARCRX_CMDC_XACT_RD43 0x1a0c -+#define EARCRX_CMDC_XACT_RD44 0x1a10 -+#define EARCRX_CMDC_XACT_RD45 0x1a14 -+#define EARCRX_CMDC_XACT_RD46 0x1a18 -+#define EARCRX_CMDC_XACT_RD47 0x1a1c -+#define EARCRX_CMDC_XACT_RD48 0x1a20 -+#define EARCRX_CMDC_XACT_RD49 0x1a24 -+#define EARCRX_CMDC_XACT_RD50 0x1a28 -+#define EARCRX_CMDC_XACT_RD51 0x1a2c -+#define EARCRX_CMDC_XACT_RD52 0x1a30 -+#define EARCRX_CMDC_XACT_RD53 0x1a34 -+#define EARCRX_CMDC_XACT_RD54 0x1a38 -+#define EARCRX_CMDC_XACT_RD55 0x1a3c -+#define EARCRX_CMDC_XACT_RD56 0x1a40 -+#define EARCRX_CMDC_XACT_RD57 0x1a44 -+#define EARCRX_CMDC_XACT_RD58 0x1a48 -+#define EARCRX_CMDC_XACT_RD59 0x1a4c -+#define EARCRX_CMDC_XACT_RD60 0x1a50 -+#define EARCRX_CMDC_XACT_RD61 0x1a54 -+#define EARCRX_CMDC_XACT_RD62 0x1a58 -+#define EARCRX_CMDC_XACT_RD63 0x1a5c -+#define EARCRX_CMDC_XACT_RD64 0x1a60 -+#define EARCRX_CMDC_SYNC_CONFIG 0x1b00 -+/* eARC RX DMAC Registers */ -+#define EARCRX_DMAC_PHY_CONTROL 0x1c00 -+#define EARCRX_DMAC_CONFIG 0x1c08 -+#define EARCRX_DMAC_CONTROL0 0x1c0c -+#define EARCRX_DMAC_AUDIO_EN BIT(1) -+#define EARCRX_DMAC_EN BIT(0) -+#define EARCRX_DMAC_CONTROL1 0x1c10 -+#define EARCRX_DMAC_STATUS 0x1c14 -+#define EARCRX_DMAC_CHSTATUS0 0x1c18 -+#define EARCRX_DMAC_CHSTATUS1 0x1c1c -+#define EARCRX_DMAC_CHSTATUS2 0x1c20 -+#define EARCRX_DMAC_CHSTATUS3 0x1c24 -+#define EARCRX_DMAC_CHSTATUS4 0x1c28 -+#define EARCRX_DMAC_CHSTATUS5 0x1c2c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC0 0x1c30 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC1 0x1c34 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC2 0x1c38 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC3 0x1c3c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC4 0x1c40 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC5 0x1c44 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC6 0x1c48 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC7 0x1c4c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC8 0x1c50 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC9 0x1c54 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC10 0x1c58 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_AC11 0x1c5c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT0 0x1c60 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT1 0x1c64 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT2 0x1c68 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT3 0x1c6c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT4 0x1c70 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT5 0x1c74 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT6 0x1c78 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT7 0x1c7c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT8 0x1c80 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT9 0x1c84 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT10 0x1c88 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC1_PKT11 0x1c8c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT0 0x1c90 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT1 0x1c94 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT2 0x1c98 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT3 0x1c9c -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT4 0x1ca0 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT5 0x1ca4 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT6 0x1ca8 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT7 0x1cac -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT8 0x1cb0 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT9 0x1cb4 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT10 0x1cb8 -+#define EARCRX_DMAC_USRDATA_MSG_HDMI_ISRC2_PKT11 0x1cbc -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC0 0x1cc0 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC1 0x1cc4 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC2 0x1cc8 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC3 0x1ccc -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC4 0x1cd0 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC5 0x1cd4 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC6 0x1cd8 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC7 0x1cdc -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC8 0x1ce0 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC9 0x1ce4 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC10 0x1ce8 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC11 0x1cec -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC12 0x1cf0 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC13 0x1cf4 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC14 0x1cf8 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC15 0x1cfc -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC16 0x1d00 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC17 0x1d04 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC18 0x1d08 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC19 0x1d0c -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC20 0x1d10 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC21 0x1d14 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC22 0x1d18 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC23 0x1d1c -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC24 0x1d20 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC25 0x1d24 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC26 0x1d28 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC27 0x1d2c -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC28 0x1d30 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC29 0x1d34 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC30 0x1d38 -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC31 0x1d3c -+#define EARCRX_DMAC_USRDATA_MSG_GENERIC32 0x1d40 -+#define EARCRX_DMAC_CHSTATUS_STREAMER0 0x1d44 -+#define EARCRX_DMAC_CHSTATUS_STREAMER1 0x1d48 -+#define EARCRX_DMAC_CHSTATUS_STREAMER2 0x1d4c -+#define EARCRX_DMAC_CHSTATUS_STREAMER3 0x1d50 -+#define EARCRX_DMAC_CHSTATUS_STREAMER4 0x1d54 -+#define EARCRX_DMAC_CHSTATUS_STREAMER5 0x1d58 -+#define EARCRX_DMAC_CHSTATUS_STREAMER6 0x1d5c -+#define EARCRX_DMAC_CHSTATUS_STREAMER7 0x1d60 -+#define EARCRX_DMAC_CHSTATUS_STREAMER8 0x1d64 -+#define EARCRX_DMAC_CHSTATUS_STREAMER9 0x1d68 -+#define EARCRX_DMAC_CHSTATUS_STREAMER10 0x1d6c -+#define EARCRX_DMAC_CHSTATUS_STREAMER11 0x1d70 -+#define EARCRX_DMAC_CHSTATUS_STREAMER12 0x1d74 -+#define EARCRX_DMAC_CHSTATUS_STREAMER13 0x1d78 -+#define EARCRX_DMAC_CHSTATUS_STREAMER14 0x1d7c -+#define EARCRX_DMAC_USRDATA_STREAMER0 0x1d80 -+/* Main Unit Interrupt Registers */ -+#define MAIN_INTVEC_INDEX 0x3000 -+#define MAINUNIT_0_INT_STATUS 0x3010 -+#define MAINUNIT_0_INT_MASK_N 0x3014 -+#define MAINUNIT_0_INT_CLEAR 0x3018 -+#define MAINUNIT_0_INT_FORCE 0x301c -+#define MAINUNIT_1_INT_STATUS 0x3020 -+#define FLT_EXIT_TO_LTSL_IRQ BIT(22) -+#define FLT_EXIT_TO_LTS4_IRQ BIT(21) -+#define FLT_EXIT_TO_LTSP_IRQ BIT(20) -+#define SCDC_NACK_RCVD_IRQ BIT(12) -+#define SCDC_RR_REPLY_STOP_IRQ BIT(11) -+#define SCDC_UPD_FLAGS_CLR_IRQ BIT(10) -+#define SCDC_UPD_FLAGS_CHG_IRQ BIT(9) -+#define SCDC_UPD_FLAGS_RD_IRQ BIT(8) -+#define I2CM_NACK_RCVD_IRQ BIT(2) -+#define I2CM_READ_REQUEST_IRQ BIT(1) -+#define I2CM_OP_DONE_IRQ BIT(0) -+#define MAINUNIT_1_INT_MASK_N 0x3024 -+#define I2CM_NACK_RCVD_MASK_N BIT(2) -+#define I2CM_READ_REQUEST_MASK_N BIT(1) -+#define I2CM_OP_DONE_MASK_N BIT(0) -+#define MAINUNIT_1_INT_CLEAR 0x3028 -+#define I2CM_NACK_RCVD_CLEAR BIT(2) -+#define I2CM_READ_REQUEST_CLEAR BIT(1) -+#define I2CM_OP_DONE_CLEAR BIT(0) -+#define MAINUNIT_1_INT_FORCE 0x302c -+/* AVPUNIT Interrupt Registers */ -+#define AVP_INTVEC_INDEX 0x3800 -+#define AVP_0_INT_STATUS 0x3810 -+#define AVP_0_INT_MASK_N 0x3814 -+#define AVP_0_INT_CLEAR 0x3818 -+#define AVP_0_INT_FORCE 0x381c -+#define AVP_1_INT_STATUS 0x3820 -+#define AVP_1_INT_MASK_N 0x3824 -+#define HDCP14_AUTH_CHG_MASK_N BIT(6) -+#define AVP_1_INT_CLEAR 0x3828 -+#define AVP_1_INT_FORCE 0x382c -+#define AVP_2_INT_STATUS 0x3830 -+#define AVP_2_INT_MASK_N 0x3834 -+#define AVP_2_INT_CLEAR 0x3838 -+#define AVP_2_INT_FORCE 0x383c -+#define AVP_3_INT_STATUS 0x3840 -+#define AVP_3_INT_MASK_N 0x3844 -+#define AVP_3_INT_CLEAR 0x3848 -+#define AVP_3_INT_FORCE 0x384c -+#define AVP_4_INT_STATUS 0x3850 -+#define AVP_4_INT_MASK_N 0x3854 -+#define AVP_4_INT_CLEAR 0x3858 -+#define AVP_4_INT_FORCE 0x385c -+#define AVP_5_INT_STATUS 0x3860 -+#define AVP_5_INT_MASK_N 0x3864 -+#define AVP_5_INT_CLEAR 0x3868 -+#define AVP_5_INT_FORCE 0x386c -+#define AVP_6_INT_STATUS 0x3870 -+#define AVP_6_INT_MASK_N 0x3874 -+#define AVP_6_INT_CLEAR 0x3878 -+#define AVP_6_INT_FORCE 0x387c -+/* CEC Interrupt Registers */ -+#define CEC_INT_STATUS 0x4000 -+#define CEC_INT_MASK_N 0x4004 -+#define CEC_INT_CLEAR 0x4008 -+#define CEC_INT_FORCE 0x400c -+/* eARC RX Interrupt Registers */ -+#define EARCRX_INTVEC_INDEX 0x4800 -+#define EARCRX_0_INT_STATUS 0x4810 -+#define EARCRX_CMDC_DISCOVERY_TIMEOUT_IRQ BIT(9) -+#define EARCRX_CMDC_DISCOVERY_DONE_IRQ BIT(8) -+#define EARCRX_0_INT_MASK_N 0x4814 -+#define EARCRX_0_INT_CLEAR 0x4818 -+#define EARCRX_0_INT_FORCE 0x481c -+#define EARCRX_1_INT_STATUS 0x4820 -+#define EARCRX_1_INT_MASK_N 0x4824 -+#define EARCRX_1_INT_CLEAR 0x4828 -+#define EARCRX_1_INT_FORCE 0x482c -+ -+#endif /* __DW_HDMI_QP_H__ */ -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c -index 0c79a9ba4..52374e943 100644 ---- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c -@@ -9,6 +9,8 @@ - #include - #include - #include -+#include -+#include - #include - #include - #include -@@ -18,6 +20,7 @@ - #include - #include - #include -+#include - - #include - -@@ -36,6 +39,7 @@ - - #include "dw-hdmi-audio.h" - #include "dw-hdmi-cec.h" -+#include "dw-hdmi-hdcp.h" - #include "dw-hdmi.h" - - #define DDC_CI_ADDR 0x37 -@@ -48,6 +52,11 @@ - - #define HDMI14_MAX_TMDSCLK 340000000 - -+static const unsigned int dw_hdmi_cable[] = { -+ EXTCON_DISP_HDMI, -+ EXTCON_NONE, -+}; -+ - enum hdmi_datamap { - RGB444_8B = 0x01, - RGB444_10B = 0x03, -@@ -62,6 +71,61 @@ enum hdmi_datamap { - YCbCr422_12B = 0x12, - }; - -+/* -+ * Unless otherwise noted, entries in this table are 100% optimization. -+ * Values can be obtained from hdmi_compute_n() but that function is -+ * slow so we pre-compute values we expect to see. -+ * -+ * All 32k and 48k values are expected to be the same (due to the way -+ * the math works) for any rate that's an exact kHz. -+ */ -+static const struct dw_hdmi_audio_tmds_n common_tmds_n_table[] = { -+ { .tmds = 25175000, .n_32k = 4096, .n_44k1 = 12854, .n_48k = 6144, }, -+ { .tmds = 25200000, .n_32k = 4096, .n_44k1 = 5656, .n_48k = 6144, }, -+ { .tmds = 27000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 28320000, .n_32k = 4096, .n_44k1 = 5586, .n_48k = 6144, }, -+ { .tmds = 30240000, .n_32k = 4096, .n_44k1 = 5642, .n_48k = 6144, }, -+ { .tmds = 31500000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, -+ { .tmds = 32000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, -+ { .tmds = 33750000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, -+ { .tmds = 36000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ { .tmds = 40000000, .n_32k = 4096, .n_44k1 = 5733, .n_48k = 6144, }, -+ { .tmds = 49500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 50000000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, -+ { .tmds = 54000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ { .tmds = 65000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 68250000, .n_32k = 4096, .n_44k1 = 5376, .n_48k = 6144, }, -+ { .tmds = 71000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 72000000, .n_32k = 4096, .n_44k1 = 5635, .n_48k = 6144, }, -+ { .tmds = 73250000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, -+ { .tmds = 74250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, -+ { .tmds = 75000000, .n_32k = 4096, .n_44k1 = 5880, .n_48k = 6144, }, -+ { .tmds = 78750000, .n_32k = 4096, .n_44k1 = 5600, .n_48k = 6144, }, -+ { .tmds = 78800000, .n_32k = 4096, .n_44k1 = 5292, .n_48k = 6144, }, -+ { .tmds = 79500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, -+ { .tmds = 83500000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 85500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 88750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, -+ { .tmds = 97750000, .n_32k = 4096, .n_44k1 = 14112, .n_48k = 6144, }, -+ { .tmds = 101000000, .n_32k = 4096, .n_44k1 = 7056, .n_48k = 6144, }, -+ { .tmds = 106500000, .n_32k = 4096, .n_44k1 = 4704, .n_48k = 6144, }, -+ { .tmds = 108000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ { .tmds = 115500000, .n_32k = 4096, .n_44k1 = 5712, .n_48k = 6144, }, -+ { .tmds = 119000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, -+ { .tmds = 135000000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 146250000, .n_32k = 4096, .n_44k1 = 6272, .n_48k = 6144, }, -+ { .tmds = 148500000, .n_32k = 4096, .n_44k1 = 5488, .n_48k = 6144, }, -+ { .tmds = 154000000, .n_32k = 4096, .n_44k1 = 5544, .n_48k = 6144, }, -+ { .tmds = 162000000, .n_32k = 4096, .n_44k1 = 5684, .n_48k = 6144, }, -+ -+ /* For 297 MHz+ HDMI spec have some other rule for setting N */ -+ { .tmds = 297000000, .n_32k = 3073, .n_44k1 = 4704, .n_48k = 5120, }, -+ { .tmds = 594000000, .n_32k = 3073, .n_44k1 = 9408, .n_48k = 10240, }, -+ -+ /* End of table */ -+ { .tmds = 0, .n_32k = 0, .n_44k1 = 0, .n_48k = 0, }, -+}; -+ - static const u16 csc_coeff_default[3][4] = { - { 0x2000, 0x0000, 0x0000, 0x0000 }, - { 0x0000, 0x2000, 0x0000, 0x0000 }, -@@ -98,12 +162,47 @@ static const u16 csc_coeff_rgb_full_to_rgb_limited[3][4] = { - { 0x0000, 0x0000, 0x1b7c, 0x0020 } - }; - -+static const struct drm_display_mode dw_hdmi_default_modes[] = { -+ /* 4 - 1280x720@60Hz 16:9 */ -+ { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390, -+ 1430, 1650, 0, 720, 725, 730, 750, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 16 - 1920x1080@60Hz 16:9 */ -+ { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008, -+ 2052, 2200, 0, 1080, 1084, 1089, 1125, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 31 - 1920x1080@50Hz 16:9 */ -+ { DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448, -+ 2492, 2640, 0, 1080, 1084, 1089, 1125, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 19 - 1280x720@50Hz 16:9 */ -+ { DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720, -+ 1760, 1980, 0, 720, 725, 730, 750, 0, -+ DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, }, -+ /* 17 - 720x576@50Hz 4:3 */ -+ { DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732, -+ 796, 864, 0, 576, 581, 586, 625, 0, -+ DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, -+ /* 2 - 720x480@60Hz 4:3 */ -+ { DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736, -+ 798, 858, 0, 480, 489, 495, 525, 0, -+ DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), -+ .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, }, -+}; -+ - struct hdmi_vmode { - bool mdataenablepolarity; - -+ unsigned int previous_pixelclock; - unsigned int mpixelclock; - unsigned int mpixelrepetitioninput; - unsigned int mpixelrepetitionoutput; -+ unsigned int previous_tmdsclock; - unsigned int mtmdsclock; - }; - -@@ -112,8 +211,8 @@ struct hdmi_data_info { - unsigned int enc_out_bus_format; - unsigned int enc_in_encoding; - unsigned int enc_out_encoding; -+ unsigned int quant_range; - unsigned int pix_repet_factor; -- unsigned int hdcp_enable; - struct hdmi_vmode video_mode; - bool rgb_limited_range; - }; -@@ -128,6 +227,9 @@ struct dw_hdmi_i2c { - u8 slave_reg; - bool is_regaddr; - bool is_segment; -+ -+ unsigned int scl_high_ns; -+ unsigned int scl_low_ns; - }; - - struct dw_hdmi_phy_data { -@@ -143,6 +245,8 @@ struct dw_hdmi_phy_data { - struct dw_hdmi { - struct drm_connector connector; - struct drm_bridge bridge; -+ struct drm_bridge *next_bridge; -+ struct platform_device *hdcp_dev; - - unsigned int version; - -@@ -156,8 +260,10 @@ struct dw_hdmi { - - struct hdmi_data_info hdmi_data; - const struct dw_hdmi_plat_data *plat_data; -+ struct dw_hdcp *hdcp; - - int vic; -+ int irq; - - u8 edid[HDMI_EDID_LEN]; - -@@ -174,6 +280,13 @@ struct dw_hdmi { - void __iomem *regs; - bool sink_is_hdmi; - bool sink_has_audio; -+ bool hpd_state; -+ bool support_hdmi; -+ bool force_logo; -+ int force_output; -+ -+ struct delayed_work work; -+ struct workqueue_struct *workqueue; - - struct pinctrl *pinctrl; - struct pinctrl_state *default_state; -@@ -190,10 +303,14 @@ struct dw_hdmi { - - spinlock_t audio_lock; - struct mutex audio_mutex; -+ struct dentry *debugfs_dir; - unsigned int sample_rate; - unsigned int audio_cts; - unsigned int audio_n; - bool audio_enable; -+ bool scramble_low_rates; -+ -+ struct extcon_dev *extcon; - - unsigned int reg_shift; - struct regmap *regm; -@@ -202,10 +319,12 @@ struct dw_hdmi { - - struct mutex cec_notifier_mutex; - struct cec_notifier *cec_notifier; -+ struct cec_adapter *cec_adap; - - hdmi_codec_plugged_cb plugged_cb; - struct device *codec_dev; - enum drm_connector_status last_connector_result; -+ bool initialized; /* hdmi is enabled before bind */ - }; - - #define HDMI_IH_PHY_STAT0_RX_SENSE \ -@@ -263,6 +382,124 @@ static void hdmi_mask_writeb(struct dw_hdmi *hdmi, u8 data, unsigned int reg, - hdmi_modb(hdmi, data << shift, mask, reg); - } - -+static bool dw_hdmi_check_output_type_changed(struct dw_hdmi *hdmi) -+{ -+ bool sink_hdmi; -+ -+ sink_hdmi = hdmi->sink_is_hdmi; -+ -+ if (hdmi->force_output == 1) -+ hdmi->sink_is_hdmi = true; -+ else if (hdmi->force_output == 2) -+ hdmi->sink_is_hdmi = false; -+ else -+ hdmi->sink_is_hdmi = hdmi->support_hdmi; -+ -+ if (sink_hdmi != hdmi->sink_is_hdmi) -+ return true; -+ -+ return false; -+} -+ -+static void repo_hpd_event(struct work_struct *p_work) -+{ -+ struct dw_hdmi *hdmi = container_of(p_work, struct dw_hdmi, work.work); -+ enum drm_connector_status status = hdmi->hpd_state ? -+ connector_status_connected : connector_status_disconnected; -+ u8 phy_stat = hdmi_readb(hdmi, HDMI_PHY_STAT0); -+ -+ mutex_lock(&hdmi->mutex); -+ if (!(phy_stat & HDMI_PHY_RX_SENSE)) -+ hdmi->rxsense = false; -+ if (phy_stat & HDMI_PHY_HPD) -+ hdmi->rxsense = true; -+ mutex_unlock(&hdmi->mutex); -+ -+ if (hdmi->bridge.dev) { -+ bool change; -+ -+ change = drm_helper_hpd_irq_event(hdmi->bridge.dev); -+ if (change && hdmi->cec_adap && -+ hdmi->cec_adap->devnode.registered) -+ cec_queue_pin_hpd_event(hdmi->cec_adap, -+ hdmi->hpd_state, -+ ktime_get()); -+ drm_bridge_hpd_notify(&hdmi->bridge, status); -+ } -+} -+ -+static bool check_hdmi_irq(struct dw_hdmi *hdmi, int intr_stat, -+ int phy_int_pol) -+{ -+ int msecs; -+ -+ /* To determine whether interrupt type is HPD */ -+ if (!(intr_stat & HDMI_IH_PHY_STAT0_HPD)) -+ return false; -+ -+ if (phy_int_pol & HDMI_PHY_HPD) { -+ dev_dbg(hdmi->dev, "dw hdmi plug in\n"); -+ msecs = 150; -+ hdmi->hpd_state = true; -+ } else { -+ dev_dbg(hdmi->dev, "dw hdmi plug out\n"); -+ msecs = 20; -+ hdmi->hpd_state = false; -+ } -+ mod_delayed_work(hdmi->workqueue, &hdmi->work, msecs_to_jiffies(msecs)); -+ -+ return true; -+} -+ -+static void init_hpd_work(struct dw_hdmi *hdmi) -+{ -+ hdmi->workqueue = create_workqueue("hpd_queue"); -+ INIT_DELAYED_WORK(&hdmi->work, repo_hpd_event); -+} -+ -+static void dw_hdmi_i2c_set_divs(struct dw_hdmi *hdmi) -+{ -+ unsigned long clk_rate_khz; -+ unsigned long low_ns, high_ns; -+ unsigned long div_low, div_high; -+ -+ /* Standard-mode */ -+ if (hdmi->i2c->scl_high_ns < 4000) -+ high_ns = 4708; -+ else -+ high_ns = hdmi->i2c->scl_high_ns; -+ -+ if (hdmi->i2c->scl_low_ns < 4700) -+ low_ns = 4916; -+ else -+ low_ns = hdmi->i2c->scl_low_ns; -+ -+ /* Adjust to avoid overflow */ -+ clk_rate_khz = DIV_ROUND_UP(clk_get_rate(hdmi->isfr_clk), 1000); -+ -+ div_low = (clk_rate_khz * low_ns) / 1000000; -+ if ((clk_rate_khz * low_ns) % 1000000) -+ div_low++; -+ -+ div_high = (clk_rate_khz * high_ns) / 1000000; -+ if ((clk_rate_khz * high_ns) % 1000000) -+ div_high++; -+ -+ /* Maximum divider supported by hw is 0xffff */ -+ if (div_low > 0xffff) -+ div_low = 0xffff; -+ -+ if (div_high > 0xffff) -+ div_high = 0xffff; -+ -+ hdmi_writeb(hdmi, div_high & 0xff, HDMI_I2CM_SS_SCL_HCNT_0_ADDR); -+ hdmi_writeb(hdmi, (div_high >> 8) & 0xff, -+ HDMI_I2CM_SS_SCL_HCNT_1_ADDR); -+ hdmi_writeb(hdmi, div_low & 0xff, HDMI_I2CM_SS_SCL_LCNT_0_ADDR); -+ hdmi_writeb(hdmi, (div_low >> 8) & 0xff, -+ HDMI_I2CM_SS_SCL_LCNT_1_ADDR); -+} -+ - static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) - { - hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL, -@@ -276,7 +513,8 @@ static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) - hdmi_writeb(hdmi, 0x00, HDMI_I2CM_SOFTRSTZ); - - /* Set Standard Mode speed (determined to be 100KHz on iMX6) */ -- hdmi_writeb(hdmi, 0x00, HDMI_I2CM_DIV); -+ hdmi_modb(hdmi, HDMI_I2CM_DIV_STD_MODE, -+ HDMI_I2CM_DIV_FAST_STD_MODE, HDMI_I2CM_DIV); - - /* Set done, not acknowledged and arbitration interrupt polarities */ - hdmi_writeb(hdmi, HDMI_I2CM_INT_DONE_POL, HDMI_I2CM_INT); -@@ -290,6 +528,11 @@ static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi) - /* Mute DONE and ERROR interrupts */ - hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE, - HDMI_IH_MUTE_I2CM_STAT0); -+ -+ /* set SDA high level holding time */ -+ hdmi_writeb(hdmi, 0x48, HDMI_I2CM_SDA_HOLD); -+ -+ dw_hdmi_i2c_set_divs(hdmi); - } - - static bool dw_hdmi_i2c_unwedge(struct dw_hdmi *hdmi) -@@ -461,6 +704,8 @@ static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap, - hdmi_writeb(hdmi, 0x00, HDMI_IH_MUTE_I2CM_STAT0); - - /* Set slave device address taken from the first I2C message */ -+ if (addr == DDC_SEGMENT_ADDR && msgs[0].len == 1) -+ addr = DDC_ADDR; - hdmi_writeb(hdmi, addr, HDMI_I2CM_SLAVE); - - /* Set slave device register address on transfer */ -@@ -570,60 +815,117 @@ static void hdmi_set_cts_n(struct dw_hdmi *hdmi, unsigned int cts, - hdmi_writeb(hdmi, n & 0xff, HDMI_AUD_N1); - } - --static unsigned int hdmi_compute_n(unsigned int freq, unsigned long pixel_clk) -+static int hdmi_match_tmds_n_table(struct dw_hdmi *hdmi, -+ unsigned long pixel_clk, -+ unsigned long freq) - { -- unsigned int n = (128 * freq) / 1000; -- unsigned int mult = 1; -+ const struct dw_hdmi_plat_data *plat_data = hdmi->plat_data; -+ const struct dw_hdmi_audio_tmds_n *tmds_n = NULL; -+ int i; -+ -+ if (plat_data->tmds_n_table) { -+ for (i = 0; plat_data->tmds_n_table[i].tmds != 0; i++) { -+ if (pixel_clk == plat_data->tmds_n_table[i].tmds) { -+ tmds_n = &plat_data->tmds_n_table[i]; -+ break; -+ } -+ } -+ } - -- while (freq > 48000) { -- mult *= 2; -- freq /= 2; -+ if (tmds_n == NULL) { -+ for (i = 0; common_tmds_n_table[i].tmds != 0; i++) { -+ if (pixel_clk == common_tmds_n_table[i].tmds) { -+ tmds_n = &common_tmds_n_table[i]; -+ break; -+ } -+ } - } - -+ if (tmds_n == NULL) -+ return -ENOENT; -+ - switch (freq) { - case 32000: -- if (pixel_clk == 25175000) -- n = 4576; -- else if (pixel_clk == 27027000) -- n = 4096; -- else if (pixel_clk == 74176000 || pixel_clk == 148352000) -- n = 11648; -- else -- n = 4096; -- n *= mult; -- break; -- -+ return tmds_n->n_32k; - case 44100: -- if (pixel_clk == 25175000) -- n = 7007; -- else if (pixel_clk == 74176000) -- n = 17836; -- else if (pixel_clk == 148352000) -- n = 8918; -- else -- n = 6272; -- n *= mult; -- break; -- -+ case 88200: -+ case 176400: -+ return (freq / 44100) * tmds_n->n_44k1; - case 48000: -- if (pixel_clk == 25175000) -- n = 6864; -- else if (pixel_clk == 27027000) -- n = 6144; -- else if (pixel_clk == 74176000) -- n = 11648; -- else if (pixel_clk == 148352000) -- n = 5824; -- else -- n = 6144; -- n *= mult; -- break; -- -+ case 96000: -+ case 192000: -+ return (freq / 48000) * tmds_n->n_48k; - default: -- break; -+ return -ENOENT; -+ } -+} -+ -+static u64 hdmi_audio_math_diff(unsigned int freq, unsigned int n, -+ unsigned int pixel_clk) -+{ -+ u64 final, diff; -+ u64 cts; -+ -+ final = (u64)pixel_clk * n; -+ -+ cts = final; -+ do_div(cts, 128 * freq); -+ -+ diff = final - (u64)cts * (128 * freq); -+ -+ return diff; -+} -+ -+static unsigned int hdmi_compute_n(struct dw_hdmi *hdmi, -+ unsigned long pixel_clk, -+ unsigned long freq) -+{ -+ unsigned int min_n = DIV_ROUND_UP((128 * freq), 1500); -+ unsigned int max_n = (128 * freq) / 300; -+ unsigned int ideal_n = (128 * freq) / 1000; -+ unsigned int best_n_distance = ideal_n; -+ unsigned int best_n = 0; -+ u64 best_diff = U64_MAX; -+ int n; -+ -+ /* If the ideal N could satisfy the audio math, then just take it */ -+ if (hdmi_audio_math_diff(freq, ideal_n, pixel_clk) == 0) -+ return ideal_n; -+ -+ for (n = min_n; n <= max_n; n++) { -+ u64 diff = hdmi_audio_math_diff(freq, n, pixel_clk); -+ -+ if (diff < best_diff || (diff == best_diff && -+ abs(n - ideal_n) < best_n_distance)) { -+ best_n = n; -+ best_diff = diff; -+ best_n_distance = abs(best_n - ideal_n); -+ } -+ -+ /* -+ * The best N already satisfy the audio math, and also be -+ * the closest value to ideal N, so just cut the loop. -+ */ -+ if ((best_diff == 0) && (abs(n - ideal_n) > best_n_distance)) -+ break; - } - -- return n; -+ return best_n; -+} -+ -+static unsigned int hdmi_find_n(struct dw_hdmi *hdmi, unsigned long pixel_clk, -+ unsigned long sample_rate) -+{ -+ int n; -+ -+ n = hdmi_match_tmds_n_table(hdmi, pixel_clk, sample_rate); -+ if (n > 0) -+ return n; -+ -+ dev_warn(hdmi->dev, "Rate %lu missing; compute N dynamically\n", -+ pixel_clk); -+ -+ return hdmi_compute_n(hdmi, pixel_clk, sample_rate); - } - - /* -@@ -654,7 +956,7 @@ static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi, - u8 config3; - u64 tmp; - -- n = hdmi_compute_n(sample_rate, pixel_clk); -+ n = hdmi_find_n(hdmi, pixel_clk, sample_rate); - - config3 = hdmi_readb(hdmi, HDMI_CONFIG3_ID); - -@@ -1005,6 +1307,15 @@ static bool is_csc_needed(struct dw_hdmi *hdmi) - is_color_space_interpolation(hdmi); - } - -+static bool is_rgb_full_to_limited_needed(struct dw_hdmi *hdmi) -+{ -+ if (hdmi->hdmi_data.quant_range == HDMI_QUANTIZATION_RANGE_LIMITED || -+ (!hdmi->hdmi_data.quant_range && hdmi->hdmi_data.rgb_limited_range)) -+ return true; -+ -+ return false; -+} -+ - static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi) - { - const u16 (*csc_coeff)[3][4] = &csc_coeff_default; -@@ -1027,7 +1338,7 @@ static void dw_hdmi_update_csc_coeffs(struct dw_hdmi *hdmi) - csc_coeff = &csc_coeff_rgb_in_eitu709; - csc_scale = 0; - } else if (is_input_rgb && is_output_rgb && -- hdmi->hdmi_data.rgb_limited_range) { -+ is_rgb_full_to_limited_needed(hdmi)) { - csc_coeff = &csc_coeff_rgb_full_to_rgb_limited; - } - -@@ -1059,7 +1370,7 @@ static void hdmi_video_csc(struct dw_hdmi *hdmi) - if (is_color_space_interpolation(hdmi)) - interpolation = HDMI_CSC_CFG_INTMODE_CHROMA_INT_FORMULA1; - else if (is_color_space_decimation(hdmi)) -- decimation = HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA3; -+ decimation = HDMI_CSC_CFG_DECMODE_CHROMA_INT_FORMULA1; - - switch (hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format)) { - case 8: -@@ -1106,7 +1417,7 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) - switch (hdmi_bus_fmt_color_depth( - hdmi->hdmi_data.enc_out_bus_format)) { - case 8: -- color_depth = 4; -+ color_depth = 0; - output_select = HDMI_VP_CONF_OUTPUT_SELECTOR_BYPASS; - break; - case 10: -@@ -1144,18 +1455,15 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) - } - - /* set the packetizer registers */ -- val = ((color_depth << HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET) & -- HDMI_VP_PR_CD_COLOR_DEPTH_MASK) | -- ((hdmi_data->pix_repet_factor << -- HDMI_VP_PR_CD_DESIRED_PR_FACTOR_OFFSET) & -- HDMI_VP_PR_CD_DESIRED_PR_FACTOR_MASK); -+ val = (color_depth << HDMI_VP_PR_CD_COLOR_DEPTH_OFFSET) & -+ HDMI_VP_PR_CD_COLOR_DEPTH_MASK; - hdmi_writeb(hdmi, val, HDMI_VP_PR_CD); - - hdmi_modb(hdmi, HDMI_VP_STUFF_PR_STUFFING_STUFFING_MODE, - HDMI_VP_STUFF_PR_STUFFING_MASK, HDMI_VP_STUFF); - - /* Data from pixel repeater block */ -- if (hdmi_data->pix_repet_factor > 1) { -+ if (hdmi_data->pix_repet_factor > 0) { - vp_conf = HDMI_VP_CONF_PR_EN_ENABLE | - HDMI_VP_CONF_BYPASS_SELECT_PIX_REPEATER; - } else { /* data from packetizer block */ -@@ -1167,8 +1475,13 @@ static void hdmi_video_packetize(struct dw_hdmi *hdmi) - HDMI_VP_CONF_PR_EN_MASK | - HDMI_VP_CONF_BYPASS_SELECT_MASK, HDMI_VP_CONF); - -- hdmi_modb(hdmi, 1 << HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET, -- HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, HDMI_VP_STUFF); -+ if ((color_depth == 5 && hdmi->previous_mode.htotal % 4) || -+ (color_depth == 6 && hdmi->previous_mode.htotal % 2)) -+ hdmi_modb(hdmi, 0, HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, -+ HDMI_VP_STUFF); -+ else -+ hdmi_modb(hdmi, 1 << HDMI_VP_STUFF_IDEFAULT_PHASE_OFFSET, -+ HDMI_VP_STUFF_IDEFAULT_PHASE_MASK, HDMI_VP_STUFF); - - hdmi_writeb(hdmi, remap_size, HDMI_VP_REMAP); - -@@ -1269,6 +1582,23 @@ static bool dw_hdmi_support_scdc(struct dw_hdmi *hdmi, - return true; - } - -+static int hdmi_phy_i2c_read(struct dw_hdmi *hdmi, unsigned char addr) -+{ -+ int val; -+ -+ hdmi_writeb(hdmi, 0xFF, HDMI_IH_I2CMPHY_STAT0); -+ hdmi_writeb(hdmi, addr, HDMI_PHY_I2CM_ADDRESS_ADDR); -+ hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_1_ADDR); -+ hdmi_writeb(hdmi, 0, HDMI_PHY_I2CM_DATAI_0_ADDR); -+ hdmi_writeb(hdmi, HDMI_PHY_I2CM_OPERATION_ADDR_READ, -+ HDMI_PHY_I2CM_OPERATION_ADDR); -+ hdmi_phy_wait_i2c_done(hdmi, 1000); -+ val = hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_1_ADDR); -+ val = (val & 0xff) << 8; -+ val += hdmi_readb(hdmi, HDMI_PHY_I2CM_DATAI_0_ADDR) & 0xff; -+ return val; -+} -+ - /* - * HDMI2.0 Specifies the following procedure for High TMDS Bit Rates: - * - The Source shall suspend transmission of the TMDS clock and data -@@ -1446,6 +1776,13 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, - const struct dw_hdmi_mpll_config *mpll_config = pdata->mpll_cfg; - const struct dw_hdmi_curr_ctrl *curr_ctrl = pdata->cur_ctr; - const struct dw_hdmi_phy_config *phy_config = pdata->phy_config; -+ unsigned int tmdsclock = hdmi->hdmi_data.video_mode.mtmdsclock; -+ unsigned int depth = -+ hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); -+ -+ if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format) && -+ pdata->mpll_cfg_420) -+ mpll_config = pdata->mpll_cfg_420; - - /* TOFIX Will need 420 specific PHY configuration tables */ - -@@ -1455,11 +1792,11 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, - break; - - for (; curr_ctrl->mpixelclock != ~0UL; curr_ctrl++) -- if (mpixelclock <= curr_ctrl->mpixelclock) -+ if (tmdsclock <= curr_ctrl->mpixelclock) - break; - - for (; phy_config->mpixelclock != ~0UL; phy_config++) -- if (mpixelclock <= phy_config->mpixelclock) -+ if (tmdsclock <= phy_config->mpixelclock) - break; - - if (mpll_config->mpixelclock == ~0UL || -@@ -1467,11 +1804,18 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, - phy_config->mpixelclock == ~0UL) - return -EINVAL; - -- dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[0].cpce, -+ if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) -+ depth = fls(depth - 8); -+ else -+ depth = 0; -+ if (depth) -+ depth--; -+ -+ dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].cpce, - HDMI_3D_TX_PHY_CPCE_CTRL); -- dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[0].gmp, -+ dw_hdmi_phy_i2c_write(hdmi, mpll_config->res[depth].gmp, - HDMI_3D_TX_PHY_GMPCTRL); -- dw_hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[0], -+ dw_hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[depth], - HDMI_3D_TX_PHY_CURRCTRL); - - dw_hdmi_phy_i2c_write(hdmi, 0, HDMI_3D_TX_PHY_PLLPHBYCTRL); -@@ -1484,10 +1828,6 @@ static int hdmi_phy_configure_dwc_hdmi_3d_tx(struct dw_hdmi *hdmi, - dw_hdmi_phy_i2c_write(hdmi, phy_config->vlev_ctr, - HDMI_3D_TX_PHY_VLEVCTRL); - -- /* Override and disable clock termination. */ -- dw_hdmi_phy_i2c_write(hdmi, HDMI_3D_TX_PHY_CKCALCTRL_OVERRIDE, -- HDMI_3D_TX_PHY_CKCALCTRL); -- - return 0; - } - -@@ -1589,14 +1929,16 @@ void dw_hdmi_phy_setup_hpd(struct dw_hdmi *hdmi, void *data) - hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE, - HDMI_IH_PHY_STAT0); - -- /* Enable cable hot plug irq. */ -- hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); -+ if (!hdmi->next_bridge) { -+ /* Enable cable hot plug irq. */ -+ hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); - -- /* Clear and unmute interrupts. */ -- hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE, -- HDMI_IH_PHY_STAT0); -- hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), -- HDMI_IH_MUTE_PHY_STAT0); -+ /* Clear and unmute interrupts. */ -+ hdmi_writeb(hdmi, HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE, -+ HDMI_IH_PHY_STAT0); -+ hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), -+ HDMI_IH_MUTE_PHY_STAT0); -+ } - } - EXPORT_SYMBOL_GPL(dw_hdmi_phy_setup_hpd); - -@@ -1612,23 +1954,36 @@ static const struct dw_hdmi_phy_ops dw_hdmi_synopsys_phy_ops = { - * HDMI TX Setup - */ - --static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi) -+static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi, -+ const struct drm_display_mode *mode) - { -- u8 de; -- -- if (hdmi->hdmi_data.video_mode.mdataenablepolarity) -- de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH; -- else -- de = HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW; -- -- /* disable rx detect */ -- hdmi_modb(hdmi, HDMI_A_HDCPCFG0_RXDETECT_DISABLE, -- HDMI_A_HDCPCFG0_RXDETECT_MASK, HDMI_A_HDCPCFG0); -- -- hdmi_modb(hdmi, de, HDMI_A_VIDPOLCFG_DATAENPOL_MASK, HDMI_A_VIDPOLCFG); -- -- hdmi_modb(hdmi, HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_DISABLE, -- HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, HDMI_A_HDCPCFG1); -+ struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; -+ u8 vsync_pol, hsync_pol, data_pol, hdmi_dvi; -+ -+ /* Configure the video polarity */ -+ vsync_pol = mode->flags & DRM_MODE_FLAG_PVSYNC ? -+ HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_HIGH : -+ HDMI_A_VIDPOLCFG_VSYNCPOL_ACTIVE_LOW; -+ hsync_pol = mode->flags & DRM_MODE_FLAG_PHSYNC ? -+ HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH : -+ HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW; -+ data_pol = vmode->mdataenablepolarity ? -+ HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_HIGH : -+ HDMI_A_VIDPOLCFG_DATAENPOL_ACTIVE_LOW; -+ hdmi_modb(hdmi, vsync_pol | hsync_pol | data_pol, -+ HDMI_A_VIDPOLCFG_VSYNCPOL_MASK | -+ HDMI_A_VIDPOLCFG_HSYNCPOL_MASK | -+ HDMI_A_VIDPOLCFG_DATAENPOL_MASK, -+ HDMI_A_VIDPOLCFG); -+ -+ /* Config the display mode */ -+ hdmi_dvi = hdmi->sink_is_hdmi ? HDMI_A_HDCPCFG0_HDMIDVI_HDMI : -+ HDMI_A_HDCPCFG0_HDMIDVI_DVI; -+ hdmi_modb(hdmi, hdmi_dvi, HDMI_A_HDCPCFG0_HDMIDVI_MASK, -+ HDMI_A_HDCPCFG0); -+ -+ if (hdmi->hdcp && hdmi->hdcp->hdcp_start) -+ hdmi->hdcp->hdcp_start(hdmi->hdcp); - } - - static void hdmi_config_AVI(struct dw_hdmi *hdmi, -@@ -1642,10 +1997,15 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, - drm_hdmi_avi_infoframe_from_display_mode(&frame, connector, mode); - - if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) { -- drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, -- hdmi->hdmi_data.rgb_limited_range ? -- HDMI_QUANTIZATION_RANGE_LIMITED : -- HDMI_QUANTIZATION_RANGE_FULL); -+ /* default range */ -+ if (!hdmi->hdmi_data.quant_range) -+ drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, -+ hdmi->hdmi_data.rgb_limited_range ? -+ HDMI_QUANTIZATION_RANGE_LIMITED : -+ HDMI_QUANTIZATION_RANGE_FULL); -+ else -+ drm_hdmi_avi_infoframe_quant_range(&frame, connector, mode, -+ hdmi->hdmi_data.quant_range); - } else { - frame.quantization_range = HDMI_QUANTIZATION_RANGE_DEFAULT; - frame.ycc_quantization_range = -@@ -1680,6 +2040,14 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, - frame.extended_colorimetry = - HDMI_EXTENDED_COLORIMETRY_XV_YCC_709; - break; -+ case V4L2_YCBCR_ENC_BT2020: -+ if (hdmi->hdmi_data.enc_in_encoding == V4L2_YCBCR_ENC_BT2020) -+ frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; -+ else -+ frame.colorimetry = HDMI_COLORIMETRY_ITU_709; -+ frame.extended_colorimetry = -+ HDMI_EXTENDED_COLORIMETRY_BT2020; -+ break; - default: /* Carries no data */ - frame.colorimetry = HDMI_COLORIMETRY_ITU_601; - frame.extended_colorimetry = -@@ -1816,17 +2184,44 @@ static void hdmi_config_drm_infoframe(struct dw_hdmi *hdmi, - const struct drm_connector *connector) - { - const struct drm_connector_state *conn_state = connector->state; -+ struct hdr_output_metadata *hdr_metadata; - struct hdmi_drm_infoframe frame; - u8 buffer[30]; - ssize_t err; - int i; - -+ /* Dynamic Range and Mastering Infoframe is introduced in v2.11a. */ -+ if (hdmi->version < 0x211a) { -+ DRM_ERROR("Not support DRM Infoframe\n"); -+ return; -+ } -+ - if (!hdmi->plat_data->use_drm_infoframe) - return; - - hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_DISABLE, - HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN); - -+ if (!hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf) { -+ DRM_DEBUG("No need to set HDR metadata in infoframe\n"); -+ return; -+ } -+ -+ if (!conn_state->hdr_output_metadata) { -+ DRM_DEBUG("source metadata not set yet\n"); -+ return; -+ } -+ -+ hdr_metadata = (struct hdr_output_metadata *) -+ conn_state->hdr_output_metadata->data; -+ -+ if (!(hdmi->connector.hdr_sink_metadata.hdmi_type1.eotf & -+ BIT(hdr_metadata->hdmi_metadata_type1.eotf))) { -+ DRM_ERROR("Not support EOTF %d\n", -+ hdr_metadata->hdmi_metadata_type1.eotf); -+ return; -+ } -+ - err = drm_hdmi_infoframe_set_hdr_metadata(&frame, conn_state); - if (err < 0) - return; -@@ -1846,51 +2241,66 @@ static void hdmi_config_drm_infoframe(struct dw_hdmi *hdmi, - hdmi_writeb(hdmi, 1, HDMI_FC_DRM_UP); - hdmi_modb(hdmi, HDMI_FC_PACKET_TX_EN_DRM_ENABLE, - HDMI_FC_PACKET_TX_EN_DRM_MASK, HDMI_FC_PACKET_TX_EN); -+ -+ DRM_DEBUG("%s eotf %d end\n", __func__, -+ hdr_metadata->hdmi_metadata_type1.eotf); - } - --static void hdmi_av_composer(struct dw_hdmi *hdmi, -- const struct drm_display_info *display, -- const struct drm_display_mode *mode) -+static unsigned int -+hdmi_get_tmdsclock(struct dw_hdmi *hdmi, unsigned long mpixelclock) - { -- u8 inv_val, bytes; -- const struct drm_hdmi_info *hdmi_info = &display->hdmi; -- struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; -- int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len; -- unsigned int vdisplay, hdisplay; -- -- vmode->mpixelclock = mode->clock * 1000; -- -- dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock); -- -- vmode->mtmdsclock = vmode->mpixelclock; -+ unsigned int tmdsclock = mpixelclock; -+ unsigned int depth = -+ hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); - - if (!hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) { -- switch (hdmi_bus_fmt_color_depth( -- hdmi->hdmi_data.enc_out_bus_format)) { -+ switch (depth) { - case 16: -- vmode->mtmdsclock = vmode->mpixelclock * 2; -+ tmdsclock = mpixelclock * 2; - break; - case 12: -- vmode->mtmdsclock = vmode->mpixelclock * 3 / 2; -+ tmdsclock = mpixelclock * 3 / 2; - break; - case 10: -- vmode->mtmdsclock = vmode->mpixelclock * 5 / 4; -+ tmdsclock = mpixelclock * 5 / 4; -+ break; -+ default: - break; - } - } - -+ return tmdsclock; -+} -+ -+static void hdmi_av_composer(struct dw_hdmi *hdmi, -+ const struct drm_display_info *display, -+ const struct drm_display_mode *mode) -+{ -+ u8 inv_val, bytes; -+ const struct drm_hdmi_info *hdmi_info = &display->hdmi; -+ struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; -+ int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len; -+ unsigned int vdisplay, hdisplay; -+ -+ vmode->previous_pixelclock = vmode->mpixelclock; -+ vmode->mpixelclock = mode->crtc_clock * 1000; -+ if ((mode->flags & DRM_MODE_FLAG_3D_MASK) == -+ DRM_MODE_FLAG_3D_FRAME_PACKING) -+ vmode->mpixelclock *= 2; -+ dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock); -+ -+ vmode->previous_tmdsclock = vmode->mtmdsclock; -+ vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, vmode->mpixelclock); - if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) - vmode->mtmdsclock /= 2; -- - dev_dbg(hdmi->dev, "final tmdsclock = %d\n", vmode->mtmdsclock); - -- /* Set up HDMI_FC_INVIDCONF */ -- inv_val = (hdmi->hdmi_data.hdcp_enable || -- (dw_hdmi_support_scdc(hdmi, display) && -- (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK || -- hdmi_info->scdc.scrambling.low_rates)) ? -- HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE : -- HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE); -+ /* Set up HDMI_FC_INVIDCONF -+ * Some display equipments require that the interval -+ * between Video Data and Data island must be at least 58 pixels, -+ * and fc_invidconf.HDCP_keepout set (1'b1) can meet the requirement. -+ */ -+ inv_val = HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE; - - inv_val |= mode->flags & DRM_MODE_FLAG_PVSYNC ? - HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH : -@@ -1956,7 +2366,8 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, - /* Scrambling Control */ - if (dw_hdmi_support_scdc(hdmi, display)) { - if (vmode->mtmdsclock > HDMI14_MAX_TMDSCLK || -- hdmi_info->scdc.scrambling.low_rates) { -+ (hdmi_info->scdc.scrambling.low_rates && -+ hdmi->scramble_low_rates)) { - /* - * HDMI2.0 Specifies the following procedure: - * After the Source Device has determined that -@@ -1990,6 +2401,8 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, - HDMI_MC_SWRSTZ); - drm_scdc_set_scrambling(hdmi->ddc, 0); - } -+ } else { -+ hdmi_writeb(hdmi, 0, HDMI_FC_SCRAMBLER_CTRL); - } - - /* Set up horizontal active pixel width */ -@@ -2047,6 +2460,12 @@ static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi) - hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE; - hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS); - -+ /* Enable pixel repetition path */ -+ if (hdmi->hdmi_data.video_mode.mpixelrepetitioninput) { -+ hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_PREPCLK_DISABLE; -+ hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS); -+ } -+ - /* Enable csc path */ - if (is_csc_needed(hdmi)) { - hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE; -@@ -2122,6 +2541,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, - const struct drm_display_mode *mode) - { - int ret; -+ void *data = hdmi->plat_data->phy_data; - - hdmi_disable_overflow_interrupts(hdmi); - -@@ -2133,48 +2553,91 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, - dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic); - } - -- if ((hdmi->vic == 6) || (hdmi->vic == 7) || -- (hdmi->vic == 21) || (hdmi->vic == 22) || -- (hdmi->vic == 2) || (hdmi->vic == 3) || -- (hdmi->vic == 17) || (hdmi->vic == 18)) -+ if (hdmi->plat_data->get_enc_out_encoding) -+ hdmi->hdmi_data.enc_out_encoding = -+ hdmi->plat_data->get_enc_out_encoding(data); -+ else if ((hdmi->vic == 6) || (hdmi->vic == 7) || -+ (hdmi->vic == 21) || (hdmi->vic == 22) || -+ (hdmi->vic == 2) || (hdmi->vic == 3) || -+ (hdmi->vic == 17) || (hdmi->vic == 18)) - hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_601; - else - hdmi->hdmi_data.enc_out_encoding = V4L2_YCBCR_ENC_709; - -- hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; -- hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; -+ if (mode->flags & DRM_MODE_FLAG_DBLCLK) { -+ hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1; -+ hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 1; -+ } else { -+ hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; -+ hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; -+ } -+ /* TOFIX: Get input format from plat data or fallback to RGB888 */ -+ if (hdmi->plat_data->get_input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->get_input_bus_format(data); -+ else if (hdmi->plat_data->input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->input_bus_format; -+ else -+ hdmi->hdmi_data.enc_in_bus_format = -+ MEDIA_BUS_FMT_RGB888_1X24; - -- if (hdmi->hdmi_data.enc_in_bus_format == MEDIA_BUS_FMT_FIXED) -- hdmi->hdmi_data.enc_in_bus_format = MEDIA_BUS_FMT_RGB888_1X24; -+ /* TOFIX: Default to RGB888 output format */ -+ if (hdmi->plat_data->get_output_bus_format) -+ hdmi->hdmi_data.enc_out_bus_format = -+ hdmi->plat_data->get_output_bus_format(data); -+ else -+ hdmi->hdmi_data.enc_out_bus_format = -+ MEDIA_BUS_FMT_RGB888_1X24; - - /* TOFIX: Get input encoding from plat data or fallback to none */ -- if (hdmi->plat_data->input_bus_encoding) -+ if (hdmi->plat_data->get_enc_in_encoding) -+ hdmi->hdmi_data.enc_in_encoding = -+ hdmi->plat_data->get_enc_in_encoding(data); -+ else if (hdmi->plat_data->input_bus_encoding) - hdmi->hdmi_data.enc_in_encoding = - hdmi->plat_data->input_bus_encoding; - else - hdmi->hdmi_data.enc_in_encoding = V4L2_YCBCR_ENC_DEFAULT; - -- if (hdmi->hdmi_data.enc_out_bus_format == MEDIA_BUS_FMT_FIXED) -- hdmi->hdmi_data.enc_out_bus_format = MEDIA_BUS_FMT_RGB888_1X24; -+ -+ if (hdmi->plat_data->get_quant_range) -+ hdmi->hdmi_data.quant_range = -+ hdmi->plat_data->get_quant_range(data); - - hdmi->hdmi_data.rgb_limited_range = hdmi->sink_is_hdmi && - drm_default_rgb_quant_range(mode) == - HDMI_QUANTIZATION_RANGE_LIMITED; - -- hdmi->hdmi_data.pix_repet_factor = 0; -- hdmi->hdmi_data.hdcp_enable = 0; -+ if (!hdmi->sink_is_hdmi) -+ hdmi->hdmi_data.quant_range = HDMI_QUANTIZATION_RANGE_FULL; -+ -+ /* -+ * According to the dw-hdmi specification 6.4.2 -+ * vp_pr_cd[3:0]: -+ * 0000b: No pixel repetition (pixel sent only once) -+ * 0001b: Pixel sent two times (pixel repeated once) -+ */ -+ hdmi->hdmi_data.pix_repet_factor = -+ (mode->flags & DRM_MODE_FLAG_DBLCLK) ? 1 : 0; - hdmi->hdmi_data.video_mode.mdataenablepolarity = true; - - /* HDMI Initialization Step B.1 */ - hdmi_av_composer(hdmi, &connector->display_info, mode); - - /* HDMI Initializateion Step B.2 */ -- ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, -- &connector->display_info, -- &hdmi->previous_mode); -- if (ret) -- return ret; -- hdmi->phy.enabled = true; -+ if (!hdmi->phy.enabled || -+ hdmi->hdmi_data.video_mode.previous_pixelclock != -+ hdmi->hdmi_data.video_mode.mpixelclock || -+ hdmi->hdmi_data.video_mode.previous_tmdsclock != -+ hdmi->hdmi_data.video_mode.mtmdsclock) { -+ ret = hdmi->phy.ops->init(hdmi, hdmi->phy.data, -+ &connector->display_info, -+ &hdmi->previous_mode); -+ if (ret) -+ return ret; -+ hdmi->phy.enabled = true; -+ } - - /* HDMI Initialization Step B.3 */ - dw_hdmi_enable_video_path(hdmi); -@@ -2202,7 +2665,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, - hdmi_video_packetize(hdmi); - hdmi_video_csc(hdmi); - hdmi_video_sample(hdmi); -- hdmi_tx_hdcp_config(hdmi); -+ hdmi_tx_hdcp_config(hdmi, mode); - - dw_hdmi_clear_overflow(hdmi); - -@@ -2278,6 +2741,8 @@ static void dw_hdmi_poweroff(struct dw_hdmi *hdmi) - hdmi->phy.enabled = false; - } - -+ if (hdmi->hdcp && hdmi->hdcp->hdcp_stop) -+ hdmi->hdcp->hdcp_stop(hdmi->hdcp); - hdmi->bridge_is_on = false; - } - -@@ -2295,6 +2760,10 @@ static void dw_hdmi_update_power(struct dw_hdmi *hdmi) - } - - if (force == DRM_FORCE_OFF) { -+ if (hdmi->initialized) { -+ hdmi->initialized = false; -+ hdmi->disabled = true; -+ } - if (hdmi->bridge_is_on) - dw_hdmi_poweroff(hdmi); - } else { -@@ -2327,8 +2796,15 @@ static enum drm_connector_status dw_hdmi_detect(struct dw_hdmi *hdmi) - { - enum drm_connector_status result; - -- result = hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); -+ if (!hdmi->force_logo) { -+ mutex_lock(&hdmi->mutex); -+ hdmi->force = DRM_FORCE_UNSPECIFIED; -+ dw_hdmi_update_power(hdmi); -+ dw_hdmi_update_phy_mask(hdmi); -+ mutex_unlock(&hdmi->mutex); -+ } - -+ result = hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); - mutex_lock(&hdmi->mutex); - if (result != hdmi->last_connector_result) { - dev_dbg(hdmi->dev, "read_hpd result: %d", result); -@@ -2338,6 +2814,11 @@ static enum drm_connector_status dw_hdmi_detect(struct dw_hdmi *hdmi) - } - mutex_unlock(&hdmi->mutex); - -+ if (result == connector_status_connected) -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, true); -+ else -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, false); -+ - return result; - } - -@@ -2358,7 +2839,7 @@ static struct edid *dw_hdmi_get_edid(struct dw_hdmi *hdmi, - dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", - edid->width_cm, edid->height_cm); - -- hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid); -+ hdmi->support_hdmi = drm_detect_hdmi_monitor(edid); - hdmi->sink_has_audio = drm_detect_monitor_audio(edid); - - return edid; -@@ -2376,21 +2857,105 @@ dw_hdmi_connector_detect(struct drm_connector *connector, bool force) - return dw_hdmi_detect(hdmi); - } - -+static int -+dw_hdmi_update_hdr_property(struct drm_connector *connector) -+{ -+ struct drm_device *dev = connector->dev; -+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, -+ connector); -+ void *data = hdmi->plat_data->phy_data; -+ const struct hdr_static_metadata *metadata = -+ &connector->hdr_sink_metadata.hdmi_type1; -+ size_t size = sizeof(*metadata); -+ struct drm_property *property; -+ struct drm_property_blob *blob; -+ int ret; -+ -+ if (hdmi->plat_data->get_hdr_property) -+ property = hdmi->plat_data->get_hdr_property(data); -+ else -+ return -EINVAL; -+ -+ if (hdmi->plat_data->get_hdr_blob) -+ blob = hdmi->plat_data->get_hdr_blob(data); -+ else -+ return -EINVAL; -+ -+ ret = drm_property_replace_global_blob(dev, &blob, size, metadata, -+ &connector->base, property); -+ return ret; -+} -+ - static int dw_hdmi_connector_get_modes(struct drm_connector *connector) - { - struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, - connector); -+ struct hdr_static_metadata *metedata = -+ &connector->hdr_sink_metadata.hdmi_type1; - struct edid *edid; -- int ret; -+ struct drm_display_mode *mode; -+ struct drm_display_info *info = &connector->display_info; -+ int i, ret = 0; - -+ memset(metedata, 0, sizeof(*metedata)); - edid = dw_hdmi_get_edid(hdmi, connector); -- if (!edid) -- return 0; -+ if (edid) { -+ dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", -+ edid->width_cm, edid->height_cm); -+ drm_connector_update_edid_property(connector, edid); -+ cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid); -+ ret = drm_add_edid_modes(connector, edid); -+ if (hdmi->plat_data->get_color_changed) -+ hdmi->plat_data->get_yuv422_format(connector, edid); -+ dw_hdmi_update_hdr_property(connector); -+ kfree(edid); -+ } else { -+ hdmi->support_hdmi = true; -+ hdmi->sink_has_audio = true; -+ for (i = 0; i < ARRAY_SIZE(dw_hdmi_default_modes); i++) { -+ const struct drm_display_mode *ptr = -+ &dw_hdmi_default_modes[i]; -+ -+ mode = drm_mode_duplicate(connector->dev, ptr); -+ if (mode) { -+ if (!i) { -+ mode->type = DRM_MODE_TYPE_PREFERRED; -+ mode->picture_aspect_ratio = -+ HDMI_PICTURE_ASPECT_NONE; -+ } -+ drm_mode_probed_add(connector, mode); -+ ret++; -+ } -+ } -+ info->edid_hdmi_dc_modes = 0; -+ info->hdmi.y420_dc_modes = 0; -+ info->color_formats = 0; -+ -+ dev_info(hdmi->dev, "failed to get edid\n"); -+ } -+ dw_hdmi_check_output_type_changed(hdmi); -+ -+ return ret; -+} -+ -+static struct drm_encoder * -+dw_hdmi_connector_best_encoder(struct drm_connector *connector) -+{ -+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, -+ connector); -+ -+ return hdmi->bridge.encoder; -+} - -- drm_connector_update_edid_property(connector, edid); -- cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid); -- ret = drm_add_edid_modes(connector, edid); -- kfree(edid); -+static bool dw_hdmi_color_changed(struct drm_connector *connector) -+{ -+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, -+ connector); -+ void *data = hdmi->plat_data->phy_data; -+ bool ret = false; -+ -+ if (hdmi->plat_data->get_color_changed) -+ ret = hdmi->plat_data->get_color_changed(data); - - return ret; - } -@@ -2419,11 +2984,54 @@ static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, - drm_atomic_get_new_connector_state(state, connector); - struct drm_crtc *crtc = new_state->crtc; - struct drm_crtc_state *crtc_state; -+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, -+ connector); -+ struct drm_display_mode *mode = NULL; -+ void *data = hdmi->plat_data->phy_data; -+ struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; -+ unsigned int in_bus_format = hdmi->hdmi_data.enc_in_bus_format; -+ unsigned int out_bus_format = hdmi->hdmi_data.enc_out_bus_format; -+ bool color_changed = false; - - if (!crtc) - return 0; - -- if (!hdr_metadata_equal(old_state, new_state)) { -+ /* -+ * If HDMI is enabled in uboot, it's need to record -+ * drm_display_mode and set phy status to enabled. -+ */ -+ if (!vmode->mpixelclock) { -+ crtc_state = drm_atomic_get_crtc_state(state, crtc); -+ if (hdmi->plat_data->get_enc_in_encoding) -+ hdmi->hdmi_data.enc_in_encoding = -+ hdmi->plat_data->get_enc_in_encoding(data); -+ if (hdmi->plat_data->get_enc_out_encoding) -+ hdmi->hdmi_data.enc_out_encoding = -+ hdmi->plat_data->get_enc_out_encoding(data); -+ if (hdmi->plat_data->get_input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->get_input_bus_format(data); -+ if (hdmi->plat_data->get_output_bus_format) -+ hdmi->hdmi_data.enc_out_bus_format = -+ hdmi->plat_data->get_output_bus_format(data); -+ -+ mode = &crtc_state->mode; -+ memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); -+ vmode->mpixelclock = mode->crtc_clock * 1000; -+ vmode->previous_pixelclock = mode->clock; -+ vmode->previous_tmdsclock = mode->clock; -+ vmode->mtmdsclock = hdmi_get_tmdsclock(hdmi, -+ vmode->mpixelclock); -+ if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) -+ vmode->mtmdsclock /= 2; -+ -+ if (in_bus_format != hdmi->hdmi_data.enc_in_bus_format || -+ out_bus_format != hdmi->hdmi_data.enc_out_bus_format) -+ color_changed = true; -+ } -+ -+ if (!hdr_metadata_equal(old_state, new_state) || -+ dw_hdmi_color_changed(connector) || color_changed) { - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); -@@ -2434,12 +3042,105 @@ static int dw_hdmi_connector_atomic_check(struct drm_connector *connector, - return 0; - } - -+static int -+dw_hdmi_atomic_connector_set_property(struct drm_connector *connector, -+ struct drm_connector_state *state, -+ struct drm_property *property, -+ uint64_t val) -+{ -+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, -+ connector); -+ const struct dw_hdmi_property_ops *ops = -+ hdmi->plat_data->property_ops; -+ -+ if (ops && ops->set_property) -+ return ops->set_property(connector, state, property, -+ val, hdmi->plat_data->phy_data); -+ else -+ return -EINVAL; -+} -+ -+static int -+dw_hdmi_atomic_connector_get_property(struct drm_connector *connector, -+ const struct drm_connector_state *state, -+ struct drm_property *property, -+ uint64_t *val) -+{ -+ struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, -+ connector); -+ const struct dw_hdmi_property_ops *ops = -+ hdmi->plat_data->property_ops; -+ -+ if (ops && ops->get_property) -+ return ops->get_property(connector, state, property, -+ val, hdmi->plat_data->phy_data); -+ else -+ return -EINVAL; -+} -+ -+static int -+dw_hdmi_connector_set_property(struct drm_connector *connector, -+ struct drm_property *property, uint64_t val) -+{ -+ return dw_hdmi_atomic_connector_set_property(connector, NULL, -+ property, val); -+} -+ -+void dw_hdmi_set_quant_range(struct dw_hdmi *hdmi) -+{ -+ if (!hdmi->bridge_is_on) -+ return; -+ -+ hdmi_writeb(hdmi, HDMI_FC_GCP_SET_AVMUTE, HDMI_FC_GCP); -+ dw_hdmi_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode); -+ hdmi_writeb(hdmi, HDMI_FC_GCP_CLEAR_AVMUTE, HDMI_FC_GCP); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_set_quant_range); -+ -+void dw_hdmi_set_output_type(struct dw_hdmi *hdmi, u64 val) -+{ -+ hdmi->force_output = val; -+ -+ if (!dw_hdmi_check_output_type_changed(hdmi)) -+ return; -+ -+ if (!hdmi->bridge_is_on) -+ return; -+ -+ hdmi_writeb(hdmi, HDMI_FC_GCP_SET_AVMUTE, HDMI_FC_GCP); -+ dw_hdmi_setup(hdmi, hdmi->curr_conn, &hdmi->previous_mode); -+ hdmi_writeb(hdmi, HDMI_FC_GCP_CLEAR_AVMUTE, HDMI_FC_GCP); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_set_output_type); -+ -+bool dw_hdmi_get_output_whether_hdmi(struct dw_hdmi *hdmi) -+{ -+ return hdmi->sink_is_hdmi; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_get_output_whether_hdmi); -+ -+int dw_hdmi_get_output_type_cap(struct dw_hdmi *hdmi) -+{ -+ return hdmi->support_hdmi; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_get_output_type_cap); -+ - static void dw_hdmi_connector_force(struct drm_connector *connector) - { - struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi, - connector); - - mutex_lock(&hdmi->mutex); -+ -+ if (hdmi->force != connector->force) { -+ if (!hdmi->disabled && connector->force == DRM_FORCE_OFF) -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, -+ false); -+ else if (hdmi->disabled && connector->force == DRM_FORCE_ON) -+ extcon_set_state_sync(hdmi->extcon, EXTCON_DISP_HDMI, -+ true); -+ } -+ - hdmi->force = connector->force; - dw_hdmi_update_power(hdmi); - dw_hdmi_update_phy_mask(hdmi); -@@ -2452,15 +3153,98 @@ static const struct drm_connector_funcs dw_hdmi_connector_funcs = { - .destroy = drm_connector_cleanup, - .force = dw_hdmi_connector_force, - .reset = drm_atomic_helper_connector_reset, -+ .set_property = dw_hdmi_connector_set_property, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, -+ .atomic_set_property = dw_hdmi_atomic_connector_set_property, -+ .atomic_get_property = dw_hdmi_atomic_connector_get_property, - }; - - static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = { - .get_modes = dw_hdmi_connector_get_modes, -+ .best_encoder = dw_hdmi_connector_best_encoder, - .atomic_check = dw_hdmi_connector_atomic_check, - }; - -+static void dw_hdmi_attach_properties(struct dw_hdmi *hdmi) -+{ -+ unsigned int color = MEDIA_BUS_FMT_RGB888_1X24; -+ int video_mapping, colorspace; -+ enum drm_connector_status connect_status = -+ hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data); -+ const struct dw_hdmi_property_ops *ops = -+ hdmi->plat_data->property_ops; -+ -+ if (connect_status == connector_status_connected) { -+ video_mapping = (hdmi_readb(hdmi, HDMI_TX_INVID0) & -+ HDMI_TX_INVID0_VIDEO_MAPPING_MASK); -+ colorspace = (hdmi_readb(hdmi, HDMI_FC_AVICONF0) & -+ HDMI_FC_AVICONF0_PIX_FMT_MASK); -+ switch (video_mapping) { -+ case 0x01: -+ color = MEDIA_BUS_FMT_RGB888_1X24; -+ break; -+ case 0x03: -+ color = MEDIA_BUS_FMT_RGB101010_1X30; -+ break; -+ case 0x09: -+ if (colorspace == HDMI_COLORSPACE_YUV420) -+ color = MEDIA_BUS_FMT_UYYVYY8_0_5X24; -+ else if (colorspace == HDMI_COLORSPACE_YUV422) -+ color = MEDIA_BUS_FMT_UYVY8_1X16; -+ else -+ color = MEDIA_BUS_FMT_YUV8_1X24; -+ break; -+ case 0x0b: -+ if (colorspace == HDMI_COLORSPACE_YUV420) -+ color = MEDIA_BUS_FMT_UYYVYY10_0_5X30; -+ else if (colorspace == HDMI_COLORSPACE_YUV422) -+ color = MEDIA_BUS_FMT_UYVY10_1X20; -+ else -+ color = MEDIA_BUS_FMT_YUV10_1X30; -+ break; -+ case 0x14: -+ color = MEDIA_BUS_FMT_UYVY10_1X20; -+ break; -+ case 0x16: -+ color = MEDIA_BUS_FMT_UYVY8_1X16; -+ break; -+ default: -+ color = MEDIA_BUS_FMT_RGB888_1X24; -+ dev_err(hdmi->dev, "unexpected mapping: 0x%x\n", -+ video_mapping); -+ } -+ -+ hdmi->hdmi_data.enc_in_bus_format = color; -+ hdmi->hdmi_data.enc_out_bus_format = color; -+ /* -+ * input format will be set as yuv444 when output -+ * format is yuv420 -+ */ -+ if (color == MEDIA_BUS_FMT_UYVY10_1X20) -+ hdmi->hdmi_data.enc_in_bus_format = -+ MEDIA_BUS_FMT_YUV10_1X30; -+ else if (color == MEDIA_BUS_FMT_UYVY8_1X16) -+ hdmi->hdmi_data.enc_in_bus_format = -+ MEDIA_BUS_FMT_YUV8_1X24; -+ } -+ -+ if (ops && ops->attach_properties) -+ return ops->attach_properties(&hdmi->connector, -+ color, hdmi->version, -+ hdmi->plat_data->phy_data); -+} -+ -+static void dw_hdmi_destroy_properties(struct dw_hdmi *hdmi) -+{ -+ const struct dw_hdmi_property_ops *ops = -+ hdmi->plat_data->property_ops; -+ -+ if (ops && ops->destroy_properties) -+ return ops->destroy_properties(&hdmi->connector, -+ hdmi->plat_data->phy_data); -+} -+ - static int dw_hdmi_connector_create(struct dw_hdmi *hdmi) - { - struct drm_connector *connector = &hdmi->connector; -@@ -2497,6 +3281,8 @@ static int dw_hdmi_connector_create(struct dw_hdmi *hdmi) - - drm_connector_attach_encoder(connector, hdmi->bridge.encoder); - -+ dw_hdmi_attach_properties(hdmi); -+ - cec_fill_conn_info_from_drm(&conn_info, connector); - - notifier = cec_notifier_conn_register(hdmi->dev, NULL, &conn_info); -@@ -2771,16 +3557,36 @@ static int dw_hdmi_bridge_atomic_check(struct drm_bridge *bridge, - struct drm_connector_state *conn_state) - { - struct dw_hdmi *hdmi = bridge->driver_private; -+ void *data = hdmi->plat_data->phy_data; - -- hdmi->hdmi_data.enc_out_bus_format = -- bridge_state->output_bus_cfg.format; -+ if (bridge_state->output_bus_cfg.format == MEDIA_BUS_FMT_FIXED) { -+ if (hdmi->plat_data->get_output_bus_format) -+ hdmi->hdmi_data.enc_out_bus_format = -+ hdmi->plat_data->get_output_bus_format(data); -+ else -+ hdmi->hdmi_data.enc_out_bus_format = -+ MEDIA_BUS_FMT_RGB888_1X24; -+ -+ if (hdmi->plat_data->get_input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->get_input_bus_format(data); -+ else if (hdmi->plat_data->input_bus_format) -+ hdmi->hdmi_data.enc_in_bus_format = -+ hdmi->plat_data->input_bus_format; -+ else -+ hdmi->hdmi_data.enc_in_bus_format = -+ MEDIA_BUS_FMT_RGB888_1X24; -+ } else { -+ hdmi->hdmi_data.enc_out_bus_format = -+ bridge_state->output_bus_cfg.format; - -- hdmi->hdmi_data.enc_in_bus_format = -- bridge_state->input_bus_cfg.format; -+ hdmi->hdmi_data.enc_in_bus_format = -+ bridge_state->input_bus_cfg.format; - -- dev_dbg(hdmi->dev, "input format 0x%04x, output format 0x%04x\n", -- bridge_state->input_bus_cfg.format, -- bridge_state->output_bus_cfg.format); -+ dev_dbg(hdmi->dev, "input format 0x%04x, output format 0x%04x\n", -+ bridge_state->input_bus_cfg.format, -+ bridge_state->output_bus_cfg.format); -+ } - - return 0; - } -@@ -2789,10 +3595,22 @@ static int dw_hdmi_bridge_attach(struct drm_bridge *bridge, - enum drm_bridge_attach_flags flags) - { - struct dw_hdmi *hdmi = bridge->driver_private; -+ int ret; - - if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) - return 0; - -+ if (hdmi->next_bridge) { -+ hdmi->next_bridge->encoder = bridge->encoder; -+ ret = drm_bridge_attach(bridge->encoder, hdmi->next_bridge, bridge, flags); -+ if (ret) { -+ DRM_ERROR("Failed to attach bridge with dw-hdmi\n"); -+ return ret; -+ } -+ -+ return 0; -+ } -+ - return dw_hdmi_connector_create(hdmi); - } - -@@ -2812,17 +3630,16 @@ dw_hdmi_bridge_mode_valid(struct drm_bridge *bridge, - const struct drm_display_mode *mode) - { - struct dw_hdmi *hdmi = bridge->driver_private; -+ struct drm_connector *connector = &hdmi->connector; - const struct dw_hdmi_plat_data *pdata = hdmi->plat_data; - enum drm_mode_status mode_status = MODE_OK; - -- /* We don't support double-clocked modes */ -- if (mode->flags & DRM_MODE_FLAG_DBLCLK) -- return MODE_BAD; -+ if (hdmi->next_bridge) -+ return MODE_OK; - - if (pdata->mode_valid) -- mode_status = pdata->mode_valid(hdmi, pdata->priv_data, info, -- mode); -- -+ mode_status = pdata->mode_valid(connector, pdata->priv_data, -+ info, mode); - return mode_status; - } - -@@ -2903,6 +3720,12 @@ static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = { - .get_edid = dw_hdmi_bridge_get_edid, - }; - -+void dw_hdmi_set_cec_adap(struct dw_hdmi *hdmi, struct cec_adapter *adap) -+{ -+ hdmi->cec_adap = adap; -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_set_cec_adap); -+ - /* ----------------------------------------------------------------------------- - * IRQ Handling - */ -@@ -2928,7 +3751,7 @@ static irqreturn_t dw_hdmi_i2c_irq(struct dw_hdmi *hdmi) - static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id) - { - struct dw_hdmi *hdmi = dev_id; -- u8 intr_stat; -+ u8 intr_stat, hdcp_stat; - irqreturn_t ret = IRQ_NONE; - - if (hdmi->i2c) -@@ -2940,6 +3763,13 @@ static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id) - return IRQ_WAKE_THREAD; - } - -+ hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT); -+ if (hdcp_stat) { -+ dev_dbg(hdmi->dev, "HDCP irq %#x\n", hdcp_stat); -+ hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK); -+ return IRQ_WAKE_THREAD; -+ } -+ - return ret; - } - -@@ -2947,7 +3777,7 @@ void dw_hdmi_setup_rx_sense(struct dw_hdmi *hdmi, bool hpd, bool rx_sense) - { - mutex_lock(&hdmi->mutex); - -- if (!hdmi->force) { -+ if (!hdmi->force && !hdmi->force_logo) { - /* - * If the RX sense status indicates we're disconnected, - * clear the software rxsense status. -@@ -2974,7 +3804,7 @@ EXPORT_SYMBOL_GPL(dw_hdmi_setup_rx_sense); - static irqreturn_t dw_hdmi_irq(int irq, void *dev_id) - { - struct dw_hdmi *hdmi = dev_id; -- u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat; -+ u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat, hdcp_stat; - - intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0); - phy_int_pol = hdmi_readb(hdmi, HDMI_PHY_POL0); -@@ -3015,25 +3845,21 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id) - } - } - -- if (intr_stat & HDMI_IH_PHY_STAT0_HPD) { -- enum drm_connector_status status = phy_int_pol & HDMI_PHY_HPD -- ? connector_status_connected -- : connector_status_disconnected; -- -- dev_dbg(hdmi->dev, "EVENT=%s\n", -- status == connector_status_connected ? -- "plugin" : "plugout"); -- -- if (hdmi->bridge.dev) { -- drm_helper_hpd_irq_event(hdmi->bridge.dev); -- drm_bridge_hpd_notify(&hdmi->bridge, status); -- } -- } -+ check_hdmi_irq(hdmi, intr_stat, phy_int_pol); - - hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0); -- hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | HDMI_IH_PHY_STAT0_RX_SENSE), -- HDMI_IH_MUTE_PHY_STAT0); -- -+ if (!hdmi->next_bridge) -+ hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | -+ HDMI_IH_PHY_STAT0_RX_SENSE), -+ HDMI_IH_MUTE_PHY_STAT0); -+ -+ hdcp_stat = hdmi_readb(hdmi, HDMI_A_APIINTSTAT); -+ if (hdcp_stat) { -+ if (hdmi->hdcp) -+ hdmi->hdcp->hdcp_isr(hdmi->hdcp, hdcp_stat); -+ hdmi_writeb(hdmi, hdcp_stat, HDMI_A_APIINTCLR); -+ hdmi_writeb(hdmi, 0x00, HDMI_A_APIINTMSK); -+ } - return IRQ_HANDLED; - } - -@@ -3167,12 +3993,363 @@ static void dw_hdmi_init_hw(struct dw_hdmi *hdmi) - * Even if we are using a separate i2c adapter doing this doesn't - * hurt. - */ -- dw_hdmi_i2c_init(hdmi); -+ if (hdmi->i2c) -+ dw_hdmi_i2c_init(hdmi); - - if (hdmi->phy.ops->setup_hpd) - hdmi->phy.ops->setup_hpd(hdmi, hdmi->phy.data); - } - -+static int dw_hdmi_status_show(struct seq_file *s, void *v) -+{ -+ struct dw_hdmi *hdmi = s->private; -+ u32 val; -+ -+ seq_puts(s, "PHY: "); -+ if (!hdmi->phy.enabled) { -+ seq_puts(s, "disabled\n"); -+ return 0; -+ } -+ seq_puts(s, "enabled\t\t\tMode: "); -+ if (hdmi->sink_is_hdmi) -+ seq_puts(s, "HDMI\n"); -+ else -+ seq_puts(s, "DVI\n"); -+ if (hdmi->hdmi_data.video_mode.mtmdsclock > 340000000) -+ val = hdmi->hdmi_data.video_mode.mtmdsclock / 4; -+ else -+ val = hdmi->hdmi_data.video_mode.mtmdsclock; -+ seq_printf(s, "Pixel Clk: %uHz\t\tTMDS Clk: %uHz\n", -+ hdmi->hdmi_data.video_mode.mpixelclock, val); -+ seq_puts(s, "Color Format: "); -+ if (hdmi_bus_fmt_is_rgb(hdmi->hdmi_data.enc_out_bus_format)) -+ seq_puts(s, "RGB"); -+ else if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format)) -+ seq_puts(s, "YUV444"); -+ else if (hdmi_bus_fmt_is_yuv422(hdmi->hdmi_data.enc_out_bus_format)) -+ seq_puts(s, "YUV422"); -+ else if (hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format)) -+ seq_puts(s, "YUV420"); -+ else -+ seq_puts(s, "UNKNOWN"); -+ val = hdmi_bus_fmt_color_depth(hdmi->hdmi_data.enc_out_bus_format); -+ seq_printf(s, "\t\tColor Depth: %d bit\n", val); -+ seq_puts(s, "Colorimetry: "); -+ switch (hdmi->hdmi_data.enc_out_encoding) { -+ case V4L2_YCBCR_ENC_601: -+ seq_puts(s, "ITU.BT601"); -+ break; -+ case V4L2_YCBCR_ENC_709: -+ seq_puts(s, "ITU.BT709"); -+ break; -+ case V4L2_YCBCR_ENC_BT2020: -+ seq_puts(s, "ITU.BT2020"); -+ break; -+ default: /* Carries no data */ -+ seq_puts(s, "ITU.BT601"); -+ break; -+ } -+ -+ seq_puts(s, "\t\tEOTF: "); -+ -+ if (hdmi->version < 0x211a) { -+ seq_puts(s, "Unsupported\n"); -+ return 0; -+ } -+ -+ val = hdmi_readb(hdmi, HDMI_FC_PACKET_TX_EN); -+ if (!(val & HDMI_FC_PACKET_TX_EN_DRM_MASK)) { -+ seq_puts(s, "Off\n"); -+ return 0; -+ } -+ -+ switch (hdmi_readb(hdmi, HDMI_FC_DRM_PB0)) { -+ case HDMI_EOTF_TRADITIONAL_GAMMA_SDR: -+ seq_puts(s, "SDR"); -+ break; -+ case HDMI_EOTF_TRADITIONAL_GAMMA_HDR: -+ seq_puts(s, "HDR"); -+ break; -+ case HDMI_EOTF_SMPTE_ST2084: -+ seq_puts(s, "ST2084"); -+ break; -+ case HDMI_EOTF_BT_2100_HLG: -+ seq_puts(s, "HLG"); -+ break; -+ default: -+ seq_puts(s, "Not Defined\n"); -+ return 0; -+ } -+ -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB3) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB2); -+ seq_printf(s, "\nx0: %d", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB5) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB4); -+ seq_printf(s, "\t\t\t\ty0: %d\n", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB7) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB6); -+ seq_printf(s, "x1: %d", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB9) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB8); -+ seq_printf(s, "\t\t\t\ty1: %d\n", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB11) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB10); -+ seq_printf(s, "x2: %d", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB13) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB12); -+ seq_printf(s, "\t\t\t\ty2: %d\n", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB15) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB14); -+ seq_printf(s, "white x: %d", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB17) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB16); -+ seq_printf(s, "\t\t\twhite y: %d\n", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB19) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB18); -+ seq_printf(s, "max lum: %d", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB21) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB20); -+ seq_printf(s, "\t\t\tmin lum: %d\n", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB23) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB22); -+ seq_printf(s, "max cll: %d", val); -+ val = hdmi_readb(hdmi, HDMI_FC_DRM_PB25) << 8; -+ val |= hdmi_readb(hdmi, HDMI_FC_DRM_PB24); -+ seq_printf(s, "\t\t\tmax fall: %d\n", val); -+ return 0; -+} -+ -+static int dw_hdmi_status_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, dw_hdmi_status_show, inode->i_private); -+} -+ -+static const struct file_operations dw_hdmi_status_fops = { -+ .owner = THIS_MODULE, -+ .open = dw_hdmi_status_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+#include -+#include -+#include -+ -+struct dw_hdmi_reg_table { -+ int reg_base; -+ int reg_end; -+}; -+ -+static const struct dw_hdmi_reg_table hdmi_reg_table[] = { -+ {HDMI_DESIGN_ID, HDMI_CONFIG3_ID}, -+ {HDMI_IH_FC_STAT0, HDMI_IH_MUTE}, -+ {HDMI_TX_INVID0, HDMI_TX_BCBDATA1}, -+ {HDMI_VP_STATUS, HDMI_VP_POL}, -+ {HDMI_FC_INVIDCONF, HDMI_FC_DBGTMDS2}, -+ {HDMI_PHY_CONF0, HDMI_PHY_POL0}, -+ {HDMI_PHY_I2CM_SLAVE_ADDR, HDMI_PHY_I2CM_FS_SCL_LCNT_0_ADDR}, -+ {HDMI_AUD_CONF0, 0x3624}, -+ {HDMI_MC_SFRDIV, HDMI_MC_HEACPHY_RST}, -+ {HDMI_CSC_CFG, HDMI_CSC_COEF_C4_LSB}, -+ {HDMI_A_HDCPCFG0, 0x52bb}, -+ {0x7800, 0x7818}, -+ {0x7900, 0x790e}, -+ {HDMI_CEC_CTRL, HDMI_CEC_WKUPCTRL}, -+ {HDMI_I2CM_SLAVE, 0x7e31}, -+}; -+ -+static int dw_hdmi_ctrl_show(struct seq_file *s, void *v) -+{ -+ struct dw_hdmi *hdmi = s->private; -+ u32 i = 0, j = 0, val = 0; -+ -+ seq_puts(s, "\n>>>hdmi_ctl reg "); -+ for (i = 0; i < 16; i++) -+ seq_printf(s, " %2x", i); -+ seq_puts(s, "\n---------------------------------------------------"); -+ -+ for (i = 0; i < ARRAY_SIZE(hdmi_reg_table); i++) { -+ for (j = hdmi_reg_table[i].reg_base; -+ j <= hdmi_reg_table[i].reg_end; j++) { -+ val = hdmi_readb(hdmi, j); -+ if ((j - hdmi_reg_table[i].reg_base) % 16 == 0) -+ seq_printf(s, "\n>>>hdmi_ctl %04x:", j); -+ seq_printf(s, " %02x", val); -+ } -+ } -+ seq_puts(s, "\n---------------------------------------------------\n"); -+ -+ return 0; -+} -+ -+static int dw_hdmi_ctrl_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, dw_hdmi_ctrl_show, inode->i_private); -+} -+ -+static ssize_t -+dw_hdmi_ctrl_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ struct dw_hdmi *hdmi = -+ ((struct seq_file *)file->private_data)->private; -+ u32 reg, val; -+ char kbuf[25]; -+ -+ if (copy_from_user(kbuf, buf, count)) -+ return -EFAULT; -+ if (sscanf(kbuf, "%x%x", ®, &val) == -1) -+ return -EFAULT; -+ if (reg > HDMI_I2CM_FS_SCL_LCNT_0_ADDR) { -+ dev_err(hdmi->dev, "it is no a hdmi register\n"); -+ return count; -+ } -+ dev_info(hdmi->dev, "/**********hdmi register config******/"); -+ dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val); -+ hdmi_writeb(hdmi, val, reg); -+ return count; -+} -+ -+static const struct file_operations dw_hdmi_ctrl_fops = { -+ .owner = THIS_MODULE, -+ .open = dw_hdmi_ctrl_open, -+ .read = seq_read, -+ .write = dw_hdmi_ctrl_write, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+static int dw_hdmi_phy_show(struct seq_file *s, void *v) -+{ -+ struct dw_hdmi *hdmi = s->private; -+ u32 i; -+ -+ seq_puts(s, "\n>>>hdmi_phy reg "); -+ for (i = 0; i < 0x28; i++) -+ seq_printf(s, "regs %02x val %04x\n", -+ i, hdmi_phy_i2c_read(hdmi, i)); -+ return 0; -+} -+ -+static int dw_hdmi_phy_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, dw_hdmi_phy_show, inode->i_private); -+} -+ -+static ssize_t -+dw_hdmi_phy_write(struct file *file, const char __user *buf, -+ size_t count, loff_t *ppos) -+{ -+ struct dw_hdmi *hdmi = -+ ((struct seq_file *)file->private_data)->private; -+ u32 reg, val; -+ char kbuf[25]; -+ -+ if (copy_from_user(kbuf, buf, count)) -+ return -EFAULT; -+ if (sscanf(kbuf, "%x%x", ®, &val) == -1) -+ return -EFAULT; -+ if (reg > 0x28) { -+ dev_err(hdmi->dev, "it is not a hdmi phy register\n"); -+ return count; -+ } -+ dev_info(hdmi->dev, "/*******hdmi phy register config******/"); -+ dev_info(hdmi->dev, "\n reg=%x val=%x\n", reg, val); -+ dw_hdmi_phy_i2c_write(hdmi, val, reg); -+ return count; -+} -+ -+static const struct file_operations dw_hdmi_phy_fops = { -+ .owner = THIS_MODULE, -+ .open = dw_hdmi_phy_open, -+ .read = seq_read, -+ .write = dw_hdmi_phy_write, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+static void dw_hdmi_register_debugfs(struct device *dev, struct dw_hdmi *hdmi) -+{ -+ hdmi->debugfs_dir = debugfs_create_dir("dw-hdmi", NULL); -+ if (IS_ERR(hdmi->debugfs_dir)) { -+ dev_err(dev, "failed to create debugfs dir!\n"); -+ return; -+ } -+ debugfs_create_file("status", 0400, hdmi->debugfs_dir, -+ hdmi, &dw_hdmi_status_fops); -+ debugfs_create_file("ctrl", 0400, hdmi->debugfs_dir, -+ hdmi, &dw_hdmi_ctrl_fops); -+ debugfs_create_file("phy", 0400, hdmi->debugfs_dir, -+ hdmi, &dw_hdmi_phy_fops); -+} -+ -+static void dw_hdmi_register_hdcp(struct device *dev, struct dw_hdmi *hdmi, -+ u32 val, bool hdcp1x_enable) -+{ -+ struct dw_hdcp hdmi_hdcp = { -+ .hdmi = hdmi, -+ .write = hdmi_writeb, -+ .read = hdmi_readb, -+ .regs = hdmi->regs, -+ .reg_io_width = val, -+ .enable = hdcp1x_enable, -+ }; -+ struct platform_device_info hdcp_device_info = { -+ .parent = dev, -+ .id = PLATFORM_DEVID_AUTO, -+ .res = NULL, -+ .num_res = 0, -+ .name = DW_HDCP_DRIVER_NAME, -+ .data = &hdmi_hdcp, -+ .size_data = sizeof(hdmi_hdcp), -+ .dma_mask = DMA_BIT_MASK(32), -+ }; -+ -+ hdmi->hdcp_dev = platform_device_register_full(&hdcp_device_info); -+ if (IS_ERR(hdmi->hdcp_dev)) -+ dev_err(dev, "failed to register hdcp!\n"); -+ else -+ hdmi->hdcp = hdmi->hdcp_dev->dev.platform_data; -+} -+ -+static int get_force_logo_property(struct dw_hdmi *hdmi) -+{ -+ struct device_node *dss; -+ struct device_node *route; -+ struct device_node *route_hdmi; -+ -+ dss = of_find_node_by_name(NULL, "display-subsystem"); -+ if (!dss) { -+ dev_err(hdmi->dev, "can't find display-subsystem\n"); -+ return -ENODEV; -+ } -+ -+ route = of_find_node_by_name(dss, "route"); -+ if (!route) { -+ dev_err(hdmi->dev, "can't find route\n"); -+ of_node_put(dss); -+ return -ENODEV; -+ } -+ of_node_put(dss); -+ -+ route_hdmi = of_find_node_by_name(route, "route-hdmi"); -+ if (!route_hdmi) { -+ dev_err(hdmi->dev, "can't find route-hdmi\n"); -+ of_node_put(route); -+ return -ENODEV; -+ } -+ of_node_put(route); -+ -+ hdmi->force_logo = -+ of_property_read_bool(route_hdmi, "force-output"); -+ -+ of_node_put(route_hdmi); -+ -+ return 0; -+} -+ - /* ----------------------------------------------------------------------------- - * Probe/remove API, used from platforms based on the DRM bridge API. - */ -@@ -3181,6 +4358,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - { - struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; -+ struct device_node *endpoint; - struct platform_device_info pdevinfo; - struct device_node *ddc_node; - struct dw_hdmi_cec_data cec; -@@ -3193,11 +4371,13 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - u8 prod_id1; - u8 config0; - u8 config3; -+ bool hdcp1x_enable = 0; - - hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL); - if (!hdmi) - return ERR_PTR(-ENOMEM); - -+ hdmi->connector.stereo_allowed = 1; - hdmi->plat_data = plat_data; - hdmi->dev = dev; - hdmi->sample_rate = 48000; -@@ -3328,7 +4508,24 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - prod_id1 & HDMI_PRODUCT_ID1_HDCP ? "with" : "without", - hdmi->phy.name); - -- dw_hdmi_init_hw(hdmi); -+ ret = get_force_logo_property(hdmi); -+ if (ret) -+ goto err_iahb; -+ -+ hdmi->initialized = false; -+ ret = hdmi_readb(hdmi, HDMI_PHY_STAT0); -+ if (((ret & HDMI_PHY_TX_PHY_LOCK) && (ret & HDMI_PHY_HPD) && -+ hdmi_readb(hdmi, HDMI_FC_EXCTRLDUR)) || hdmi->force_logo) { -+ hdmi->mc_clkdis = hdmi_readb(hdmi, HDMI_MC_CLKDIS); -+ hdmi->disabled = false; -+ hdmi->bridge_is_on = true; -+ hdmi->phy.enabled = true; -+ hdmi->initialized = true; -+ } else if (ret & HDMI_PHY_TX_PHY_LOCK) { -+ hdmi->phy.ops->disable(hdmi, hdmi->phy.data); -+ } -+ -+ init_hpd_work(hdmi); - - irq = platform_get_irq(pdev, 0); - if (irq < 0) { -@@ -3336,6 +4533,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - goto err_iahb; - } - -+ hdmi->irq = irq; - ret = devm_request_threaded_irq(dev, irq, dw_hdmi_hardirq, - dw_hdmi_irq, IRQF_SHARED, - dev_name(dev), hdmi); -@@ -3371,8 +4569,20 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - hdmi->ddc = dw_hdmi_i2c_adapter(hdmi); - if (IS_ERR(hdmi->ddc)) - hdmi->ddc = NULL; -+ /* -+ * Read high and low time from device tree. If not available use -+ * the default timing scl clock rate is about 99.6KHz. -+ */ -+ if (of_property_read_u32(np, "ddc-i2c-scl-high-time-ns", -+ &hdmi->i2c->scl_high_ns)) -+ hdmi->i2c->scl_high_ns = 4708; -+ if (of_property_read_u32(np, "ddc-i2c-scl-low-time-ns", -+ &hdmi->i2c->scl_low_ns)) -+ hdmi->i2c->scl_low_ns = 4916; - } - -+ dw_hdmi_init_hw(hdmi); -+ - hdmi->bridge.driver_private = hdmi; - hdmi->bridge.funcs = &dw_hdmi_bridge_funcs; - hdmi->bridge.ops = DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID -@@ -3381,6 +4591,30 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - hdmi->bridge.of_node = pdev->dev.of_node; - #endif - -+ endpoint = of_graph_get_endpoint_by_regs(hdmi->dev->of_node, 1, -1); -+ if (endpoint && of_device_is_available(endpoint)) { -+ struct device_node *remote; -+ -+ remote = of_graph_get_remote_port_parent(endpoint); -+ of_node_put(endpoint); -+ if (!remote || !of_device_is_available(remote)) { -+ of_node_put(remote); -+ ret = -ENODEV; -+ goto err_iahb; -+ } -+ -+ hdmi->next_bridge = of_drm_find_bridge(remote); -+ of_node_put(remote); -+ if (!hdmi->next_bridge) { -+ dev_err(hdmi->dev, "can't find next bridge\n"); -+ ret = -EPROBE_DEFER; -+ goto err_iahb; -+ } -+ -+ hdmi->sink_is_hdmi = true; -+ hdmi->sink_has_audio = true; -+ } -+ - memset(&pdevinfo, 0, sizeof(pdevinfo)); - pdevinfo.parent = dev; - pdevinfo.id = PLATFORM_DEVID_AUTO; -@@ -3434,8 +4668,40 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - hdmi->cec = platform_device_register_full(&pdevinfo); - } - -+ hdmi->extcon = devm_extcon_dev_allocate(hdmi->dev, dw_hdmi_cable); -+ if (IS_ERR(hdmi->extcon)) { -+ ret = PTR_ERR(hdmi->extcon); -+ dev_err(hdmi->dev, "allocate extcon failed: %d\n", ret); -+ goto err_iahb; -+ } -+ -+ ret = devm_extcon_dev_register(hdmi->dev, hdmi->extcon); -+ if (ret) { -+ dev_err(hdmi->dev, "failed to register extcon: %d\n", -+ ret); -+ goto err_iahb; -+ } -+ -+ ret = extcon_set_property_capability(hdmi->extcon, EXTCON_DISP_HDMI, -+ EXTCON_PROP_DISP_HPD); -+ if (ret) { -+ dev_err(hdmi->dev, -+ "failed to set USB property capability: %d\n", -+ ret); -+ goto err_iahb; -+ } -+ - drm_bridge_add(&hdmi->bridge); - -+ dw_hdmi_register_debugfs(dev, hdmi); -+ -+ if (of_property_read_bool(np, "scramble-low-rates")) -+ hdmi->scramble_low_rates = true; -+ -+ if (of_property_read_bool(np, "hdcp1x-enable")) -+ hdcp1x_enable = 1; -+ dw_hdmi_register_hdcp(dev, hdmi, val, hdcp1x_enable); -+ - return hdmi; - - err_iahb: -@@ -3445,7 +4711,10 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, - err_isfr: - clk_disable_unprepare(hdmi->isfr_clk); - err_res: -- i2c_put_adapter(hdmi->ddc); -+ if (hdmi->i2c) -+ i2c_del_adapter(&hdmi->i2c->adap); -+ else -+ i2c_put_adapter(hdmi->ddc); - - return ERR_PTR(ret); - } -@@ -3453,16 +4722,35 @@ EXPORT_SYMBOL_GPL(dw_hdmi_probe); - - void dw_hdmi_remove(struct dw_hdmi *hdmi) - { -+ if (hdmi->irq) -+ disable_irq(hdmi->irq); -+ -+ cancel_delayed_work(&hdmi->work); -+ flush_workqueue(hdmi->workqueue); -+ destroy_workqueue(hdmi->workqueue); -+ -+ debugfs_remove_recursive(hdmi->debugfs_dir); -+ - drm_bridge_remove(&hdmi->bridge); - - if (hdmi->audio && !IS_ERR(hdmi->audio)) - platform_device_unregister(hdmi->audio); -+ if (hdmi->hdcp_dev && !IS_ERR(hdmi->hdcp_dev)) -+ platform_device_unregister(hdmi->hdcp_dev); - if (!IS_ERR(hdmi->cec)) - platform_device_unregister(hdmi->cec); - - /* Disable all interrupts */ - hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0); - -+ if (!hdmi->next_bridge) { -+ dw_hdmi_destroy_properties(hdmi); -+ hdmi->connector.funcs->destroy(&hdmi->connector); -+ } -+ -+ if (hdmi->bridge.encoder) -+ hdmi->bridge.encoder->funcs->destroy(hdmi->bridge.encoder); -+ - clk_disable_unprepare(hdmi->iahb_clk); - clk_disable_unprepare(hdmi->isfr_clk); - if (hdmi->cec_clk) -@@ -3480,7 +4768,7 @@ EXPORT_SYMBOL_GPL(dw_hdmi_remove); - */ - struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev, - struct drm_encoder *encoder, -- const struct dw_hdmi_plat_data *plat_data) -+ struct dw_hdmi_plat_data *plat_data) - { - struct dw_hdmi *hdmi; - int ret; -@@ -3496,6 +4784,9 @@ struct dw_hdmi *dw_hdmi_bind(struct platform_device *pdev, - return ERR_PTR(ret); - } - -+ if (!hdmi->next_bridge) -+ plat_data->connector = &hdmi->connector; -+ - return hdmi; - } - EXPORT_SYMBOL_GPL(dw_hdmi_bind); -@@ -3506,9 +4797,87 @@ void dw_hdmi_unbind(struct dw_hdmi *hdmi) - } - EXPORT_SYMBOL_GPL(dw_hdmi_unbind); - -+static void dw_hdmi_reg_initial(struct dw_hdmi *hdmi) -+{ -+ if (hdmi_readb(hdmi, HDMI_IH_MUTE)) { -+ initialize_hdmi_ih_mutes(hdmi); -+ /* unmute cec irq */ -+ hdmi_writeb(hdmi, 0x68, HDMI_IH_MUTE_CEC_STAT0); -+ -+ hdmi_writeb(hdmi, HDMI_PHY_I2CM_INT_ADDR_DONE_POL, -+ HDMI_PHY_I2CM_INT_ADDR); -+ -+ hdmi_writeb(hdmi, HDMI_PHY_I2CM_CTLINT_ADDR_NAC_POL | -+ HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL, -+ HDMI_PHY_I2CM_CTLINT_ADDR); -+ -+ if (!hdmi->next_bridge) { -+ hdmi_writeb(hdmi, HDMI_PHY_HPD | HDMI_PHY_RX_SENSE, -+ HDMI_PHY_POL0); -+ hdmi_writeb(hdmi, hdmi->phy_mask, HDMI_PHY_MASK0); -+ hdmi_writeb(hdmi, ~(HDMI_IH_PHY_STAT0_HPD | -+ HDMI_IH_PHY_STAT0_RX_SENSE), -+ HDMI_IH_MUTE_PHY_STAT0); -+ } -+ } -+} -+ -+void dw_hdmi_suspend(struct dw_hdmi *hdmi) -+{ -+ if (!hdmi) -+ return; -+ -+ mutex_lock(&hdmi->mutex); -+ -+ /* -+ * When system shutdown, hdmi should be disabled. -+ * When system suspend, dw_hdmi_bridge_disable will disable hdmi first. -+ * To prevent duplicate operation, we should determine whether hdmi -+ * has been disabled. -+ */ -+ if (!hdmi->disabled) { -+ hdmi->disabled = true; -+ dw_hdmi_update_power(hdmi); -+ dw_hdmi_update_phy_mask(hdmi); -+ } -+ mutex_unlock(&hdmi->mutex); -+ -+ if (hdmi->irq) -+ disable_irq(hdmi->irq); -+ cancel_delayed_work(&hdmi->work); -+ flush_workqueue(hdmi->workqueue); -+ pinctrl_pm_select_sleep_state(hdmi->dev); -+} -+EXPORT_SYMBOL_GPL(dw_hdmi_suspend); -+ - void dw_hdmi_resume(struct dw_hdmi *hdmi) - { -- dw_hdmi_init_hw(hdmi); -+ if (!hdmi) -+ return; -+ -+ pinctrl_pm_select_default_state(hdmi->dev); -+ mutex_lock(&hdmi->mutex); -+ dw_hdmi_reg_initial(hdmi); -+ if (hdmi->i2c) -+ dw_hdmi_i2c_init(hdmi); -+ if (hdmi->irq) -+ enable_irq(hdmi->irq); -+ /* -+ * HDMI status maybe incorrect in the following condition: -+ * HDMI plug in -> system sleep -> HDMI plug out -> system wake up. -+ * At this time, cat /sys/class/drm/card 0-HDMI-A-1/status is connected. -+ * There is no hpd interrupt, because HDMI is powerdown during suspend. -+ * So we need check the current HDMI status in this case. -+ */ -+ if (hdmi->connector.status == connector_status_connected) { -+ if (hdmi->phy.ops->read_hpd(hdmi, hdmi->phy.data) == -+ connector_status_disconnected) { -+ hdmi->hpd_state = false; -+ mod_delayed_work(hdmi->workqueue, &hdmi->work, -+ msecs_to_jiffies(20)); -+ } -+ } -+ mutex_unlock(&hdmi->mutex); - } - EXPORT_SYMBOL_GPL(dw_hdmi_resume); - -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h -index 1999db05b..509732800 100644 ---- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h -+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h -@@ -509,6 +509,51 @@ - #define HDMI_A_PRESETUP 0x501A - #define HDMI_A_SRM_BASE 0x5020 - -+/* CEC Engine Registers */ -+#define HDMI_CEC_CTRL 0x7D00 -+#define HDMI_CEC_STAT 0x7D01 -+#define HDMI_CEC_MASK 0x7D02 -+#define HDMI_CEC_POLARITY 0x7D03 -+#define HDMI_CEC_INT 0x7D04 -+#define HDMI_CEC_ADDR_L 0x7D05 -+#define HDMI_CEC_ADDR_H 0x7D06 -+#define HDMI_CEC_TX_CNT 0x7D07 -+#define HDMI_CEC_RX_CNT 0x7D08 -+#define HDMI_CEC_TX_DATA0 0x7D10 -+#define HDMI_CEC_TX_DATA1 0x7D11 -+#define HDMI_CEC_TX_DATA2 0x7D12 -+#define HDMI_CEC_TX_DATA3 0x7D13 -+#define HDMI_CEC_TX_DATA4 0x7D14 -+#define HDMI_CEC_TX_DATA5 0x7D15 -+#define HDMI_CEC_TX_DATA6 0x7D16 -+#define HDMI_CEC_TX_DATA7 0x7D17 -+#define HDMI_CEC_TX_DATA8 0x7D18 -+#define HDMI_CEC_TX_DATA9 0x7D19 -+#define HDMI_CEC_TX_DATA10 0x7D1a -+#define HDMI_CEC_TX_DATA11 0x7D1b -+#define HDMI_CEC_TX_DATA12 0x7D1c -+#define HDMI_CEC_TX_DATA13 0x7D1d -+#define HDMI_CEC_TX_DATA14 0x7D1e -+#define HDMI_CEC_TX_DATA15 0x7D1f -+#define HDMI_CEC_RX_DATA0 0x7D20 -+#define HDMI_CEC_RX_DATA1 0x7D21 -+#define HDMI_CEC_RX_DATA2 0x7D22 -+#define HDMI_CEC_RX_DATA3 0x7D23 -+#define HDMI_CEC_RX_DATA4 0x7D24 -+#define HDMI_CEC_RX_DATA5 0x7D25 -+#define HDMI_CEC_RX_DATA6 0x7D26 -+#define HDMI_CEC_RX_DATA7 0x7D27 -+#define HDMI_CEC_RX_DATA8 0x7D28 -+#define HDMI_CEC_RX_DATA9 0x7D29 -+#define HDMI_CEC_RX_DATA10 0x7D2a -+#define HDMI_CEC_RX_DATA11 0x7D2b -+#define HDMI_CEC_RX_DATA12 0x7D2c -+#define HDMI_CEC_RX_DATA13 0x7D2d -+#define HDMI_CEC_RX_DATA14 0x7D2e -+#define HDMI_CEC_RX_DATA15 0x7D2f -+#define HDMI_CEC_LOCK 0x7D30 -+#define HDMI_CEC_WKUPCTRL 0x7D31 -+ - /* I2C Master Registers (E-DDC) */ - #define HDMI_I2CM_SLAVE 0x7E00 - #define HDMI_I2CM_ADDRESS 0x7E01 -@@ -529,6 +574,7 @@ - #define HDMI_I2CM_FS_SCL_HCNT_0_ADDR 0x7E10 - #define HDMI_I2CM_FS_SCL_LCNT_1_ADDR 0x7E11 - #define HDMI_I2CM_FS_SCL_LCNT_0_ADDR 0x7E12 -+#define HDMI_I2CM_SDA_HOLD 0x7E13 - - enum { - /* PRODUCT_ID0 field values */ -@@ -842,6 +888,10 @@ enum { - HDMI_FC_AVICONF3_QUANT_RANGE_LIMITED = 0x00, - HDMI_FC_AVICONF3_QUANT_RANGE_FULL = 0x04, - -+/* HDMI_FC_GCP */ -+ HDMI_FC_GCP_SET_AVMUTE = 0x2, -+ HDMI_FC_GCP_CLEAR_AVMUTE = 0x1, -+ - /* FC_DBGFORCE field values */ - HDMI_FC_DBGFORCE_FORCEAUDIO = 0x10, - HDMI_FC_DBGFORCE_FORCEVIDEO = 0x1, -@@ -1085,6 +1135,11 @@ enum { - HDMI_I2CM_CTLINT_NAC_MASK = 0x40, - HDMI_I2CM_CTLINT_ARB_POL = 0x8, - HDMI_I2CM_CTLINT_ARB_MASK = 0x4, -+ -+/* I2CM_DIV field values */ -+ HDMI_I2CM_DIV_FAST_STD_MODE = 0x8, -+ HDMI_I2CM_DIV_FAST_MODE = 0x8, -+ HDMI_I2CM_DIV_STD_MODE = 0, - }; - - /* -diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c -index 6b268f944..fa00e24c1 100644 ---- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c -+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c -@@ -244,7 +244,7 @@ struct dw_mipi_dsi { - struct device *dev; - void __iomem *base; - -- struct clk *pclk; -+ struct reset_control *apb_rst; - - unsigned int lane_mbps; /* per lane */ - u32 channel; -@@ -316,15 +316,10 @@ static int dw_mipi_dsi_host_attach(struct mipi_dsi_host *host, - const struct dw_mipi_dsi_plat_data *pdata = dsi->plat_data; - struct drm_bridge *bridge; - struct drm_panel *panel; -+ int max_data_lanes = dsi->plat_data->max_data_lanes; - int ret; - -- if (device->lanes > dsi->plat_data->max_data_lanes) { -- dev_err(dsi->dev, "the number of data lanes(%u) is too many\n", -- device->lanes); -- return -EINVAL; -- } -- -- dsi->lanes = device->lanes; -+ dsi->lanes = (device->lanes > max_data_lanes) ? device->lanes / 2 : device->lanes; - dsi->channel = device->channel; - dsi->format = device->format; - dsi->mode_flags = device->mode_flags; -@@ -599,8 +594,14 @@ static void dw_mipi_dsi_set_mode(struct dw_mipi_dsi *dsi, - - static void dw_mipi_dsi_disable(struct dw_mipi_dsi *dsi) - { -+ const struct dw_mipi_dsi_phy_ops *phy_ops = dsi->plat_data->phy_ops; -+ -+ if (phy_ops->power_off) -+ phy_ops->power_off(dsi->plat_data->priv_data); -+ - dsi_write(dsi, DSI_PWR_UP, RESET); - dsi_write(dsi, DSI_PHY_RSTZ, PHY_RSTZ); -+ pm_runtime_put(dsi->dev); - } - - static void dw_mipi_dsi_init(struct dw_mipi_dsi *dsi) -@@ -715,16 +716,16 @@ static u32 dw_mipi_dsi_get_hcomponent_lbcc(struct dw_mipi_dsi *dsi, - const struct drm_display_mode *mode, - u32 hcomponent) - { -- u32 frac, lbcc; -+ u32 lbcc; - - lbcc = hcomponent * dsi->lane_mbps * MSEC_PER_SEC / 8; - -- frac = lbcc % mode->clock; -- lbcc = lbcc / mode->clock; -- if (frac) -- lbcc++; -+ if (mode->clock == 0) { -+ DRM_ERROR("dsi mode clock is 0!\n"); -+ return 0; -+ } - -- return lbcc; -+ return DIV_ROUND_CLOSEST_ULL(lbcc, mode->clock); - } - - static void dw_mipi_dsi_line_timer_config(struct dw_mipi_dsi *dsi, -@@ -837,13 +838,13 @@ static void dw_mipi_dsi_dphy_enable(struct dw_mipi_dsi *dsi) - ret = readl_poll_timeout(dsi->base + DSI_PHY_STATUS, val, - val & PHY_LOCK, 1000, PHY_STATUS_TIMEOUT_US); - if (ret) -- DRM_DEBUG_DRIVER("failed to wait phy lock state\n"); -+ DRM_ERROR("failed to wait phy lock state\n"); - - ret = readl_poll_timeout(dsi->base + DSI_PHY_STATUS, - val, val & PHY_STOP_STATE_CLK_LANE, 1000, - PHY_STATUS_TIMEOUT_US); - if (ret) -- DRM_DEBUG_DRIVER("failed to wait phy clk lane stop state\n"); -+ DRM_ERROR("failed to wait phy clk lane stop state\n"); - } - - static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi) -@@ -857,7 +858,6 @@ static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi) - static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge) - { - struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge); -- const struct dw_mipi_dsi_phy_ops *phy_ops = dsi->plat_data->phy_ops; - - /* - * Switch to command mode before panel-bridge post_disable & -@@ -866,6 +866,8 @@ static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge) - * before by the drm framework. - */ - dw_mipi_dsi_set_mode(dsi, 0); -+ if (dsi->slave) -+ dw_mipi_dsi_set_mode(dsi->slave, 0); - - /* - * TODO Only way found to call panel-bridge post_disable & -@@ -876,18 +878,10 @@ static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge) - if (dsi->panel_bridge->funcs->post_disable) - dsi->panel_bridge->funcs->post_disable(dsi->panel_bridge); - -- if (phy_ops->power_off) -- phy_ops->power_off(dsi->plat_data->priv_data); -- -- if (dsi->slave) { -+ if (dsi->slave) - dw_mipi_dsi_disable(dsi->slave); -- clk_disable_unprepare(dsi->slave->pclk); -- pm_runtime_put(dsi->slave->dev); -- } -- dw_mipi_dsi_disable(dsi); - -- clk_disable_unprepare(dsi->pclk); -- pm_runtime_put(dsi->dev); -+ dw_mipi_dsi_disable(dsi); - } - - static unsigned int dw_mipi_dsi_get_lanes(struct dw_mipi_dsi *dsi) -@@ -912,7 +906,11 @@ static void dw_mipi_dsi_mode_set(struct dw_mipi_dsi *dsi, - int ret; - u32 lanes = dw_mipi_dsi_get_lanes(dsi); - -- clk_prepare_enable(dsi->pclk); -+ if (dsi->apb_rst) { -+ reset_control_assert(dsi->apb_rst); -+ usleep_range(10, 20); -+ reset_control_deassert(dsi->apb_rst); -+ } - - ret = phy_ops->get_lane_mbps(priv_data, adjusted_mode, dsi->mode_flags, - lanes, dsi->format, &dsi->lane_mbps); -@@ -939,15 +937,15 @@ static void dw_mipi_dsi_mode_set(struct dw_mipi_dsi *dsi, - if (ret) - DRM_DEBUG_DRIVER("Phy init() failed\n"); - -+ if (phy_ops->power_on) -+ phy_ops->power_on(dsi->plat_data->priv_data); -+ - dw_mipi_dsi_dphy_enable(dsi); - - dw_mipi_dsi_wait_for_two_frames(adjusted_mode); - - /* Switch to cmd mode for panel-bridge pre_enable & panel prepare */ - dw_mipi_dsi_set_mode(dsi, 0); -- -- if (phy_ops->power_on) -- phy_ops->power_on(dsi->plat_data->priv_data); - } - - static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge, -@@ -959,16 +957,25 @@ static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge, - dw_mipi_dsi_mode_set(dsi, adjusted_mode); - if (dsi->slave) - dw_mipi_dsi_mode_set(dsi->slave, adjusted_mode); -+ -+ DRM_DEV_INFO(dsi->dev, "final DSI-Link bandwidth: %u x %d Mbps\n", -+ dsi->lane_mbps, dsi->slave ? dsi->lanes * 2 : dsi->lanes); - } - - static void dw_mipi_dsi_bridge_enable(struct drm_bridge *bridge) - { - struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge); - -- /* Switch to video mode for panel-bridge enable & panel enable */ -- dw_mipi_dsi_set_mode(dsi, MIPI_DSI_MODE_VIDEO); -- if (dsi->slave) -- dw_mipi_dsi_set_mode(dsi->slave, MIPI_DSI_MODE_VIDEO); -+ /* Switch to video/cmd mode for panel-bridge enable & panel enable */ -+ if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) { -+ dw_mipi_dsi_set_mode(dsi, MIPI_DSI_MODE_VIDEO); -+ if (dsi->slave) -+ dw_mipi_dsi_set_mode(dsi->slave, MIPI_DSI_MODE_VIDEO); -+ } else { -+ dw_mipi_dsi_set_mode(dsi, 0); -+ if (dsi->slave) -+ dw_mipi_dsi_set_mode(dsi->slave, 0); -+ } - } - - static enum drm_mode_status -@@ -1103,7 +1110,6 @@ __dw_mipi_dsi_probe(struct platform_device *pdev, - const struct dw_mipi_dsi_plat_data *plat_data) - { - struct device *dev = &pdev->dev; -- struct reset_control *apb_rst; - struct dw_mipi_dsi *dsi; - int ret; - -@@ -1129,20 +1135,13 @@ __dw_mipi_dsi_probe(struct platform_device *pdev, - dsi->base = plat_data->base; - } - -- dsi->pclk = devm_clk_get(dev, "pclk"); -- if (IS_ERR(dsi->pclk)) { -- ret = PTR_ERR(dsi->pclk); -- dev_err(dev, "Unable to get pclk: %d\n", ret); -- return ERR_PTR(ret); -- } -- - /* - * Note that the reset was not defined in the initial device tree, so - * we have to be prepared for it not being found. - */ -- apb_rst = devm_reset_control_get_optional_exclusive(dev, "apb"); -- if (IS_ERR(apb_rst)) { -- ret = PTR_ERR(apb_rst); -+ dsi->apb_rst = devm_reset_control_get_optional_exclusive(dev, "apb"); -+ if (IS_ERR(dsi->apb_rst)) { -+ ret = PTR_ERR(dsi->apb_rst); - - if (ret != -EPROBE_DEFER) - dev_err(dev, "Unable to get reset control: %d\n", ret); -@@ -1150,20 +1149,6 @@ __dw_mipi_dsi_probe(struct platform_device *pdev, - return ERR_PTR(ret); - } - -- if (apb_rst) { -- ret = clk_prepare_enable(dsi->pclk); -- if (ret) { -- dev_err(dev, "%s: Failed to enable pclk\n", __func__); -- return ERR_PTR(ret); -- } -- -- reset_control_assert(apb_rst); -- usleep_range(10, 20); -- reset_control_deassert(apb_rst); -- -- clk_disable_unprepare(dsi->pclk); -- } -- - dw_mipi_dsi_debugfs_init(dsi); - pm_runtime_enable(dev); - -@@ -1246,6 +1231,12 @@ void dw_mipi_dsi_unbind(struct dw_mipi_dsi *dsi) - } - EXPORT_SYMBOL_GPL(dw_mipi_dsi_unbind); - -+struct drm_connector *dw_mipi_dsi_get_connector(struct dw_mipi_dsi *dsi) -+{ -+ return drm_panel_bridge_connector(dsi->panel_bridge); -+} -+EXPORT_SYMBOL_GPL(dw_mipi_dsi_get_connector); -+ - MODULE_AUTHOR("Chris Zhong "); - MODULE_AUTHOR("Philippe Cornu "); - MODULE_DESCRIPTION("DW MIPI DSI host controller driver"); -diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c -index 8a871e5c3..9418d0847 100644 ---- a/drivers/gpu/drm/drm_atomic_helper.c -+++ b/drivers/gpu/drm/drm_atomic_helper.c -@@ -296,12 +296,14 @@ update_connector_routing(struct drm_atomic_state *state, - if (old_connector_state->crtc != new_connector_state->crtc) { - if (old_connector_state->crtc) { - crtc_state = drm_atomic_get_new_crtc_state(state, old_connector_state->crtc); -- crtc_state->connectors_changed = true; -+ if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) -+ crtc_state->connectors_changed = true; - } - - if (new_connector_state->crtc) { - crtc_state = drm_atomic_get_new_crtc_state(state, new_connector_state->crtc); -- crtc_state->connectors_changed = true; -+ if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) -+ crtc_state->connectors_changed = true; - } - } - -@@ -386,7 +388,8 @@ update_connector_routing(struct drm_atomic_state *state, - - set_best_encoder(state, new_connector_state, new_encoder); - -- crtc_state->connectors_changed = true; -+ if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) -+ crtc_state->connectors_changed = true; - - DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] using [ENCODER:%d:%s] on [CRTC:%d:%s]\n", - connector->base.id, -@@ -3544,6 +3547,9 @@ int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc, - replaced = drm_property_replace_blob(&crtc_state->degamma_lut, NULL); - replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); - replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, blob); -+#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) -+ replaced |= drm_property_replace_blob(&crtc_state->cubic_lut, NULL); -+#endif - crtc_state->color_mgmt_changed |= replaced; - - ret = drm_atomic_commit(state); -diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c -index 9ad740451..c29183d2a 100644 ---- a/drivers/gpu/drm/drm_atomic_state_helper.c -+++ b/drivers/gpu/drm/drm_atomic_state_helper.c -@@ -141,6 +141,10 @@ void __drm_atomic_helper_crtc_duplicate_state(struct drm_crtc *crtc, - drm_property_blob_get(state->ctm); - if (state->gamma_lut) - drm_property_blob_get(state->gamma_lut); -+#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) -+ if (state->cubic_lut) -+ drm_property_blob_get(state->cubic_lut); -+#endif - state->mode_changed = false; - state->active_changed = false; - state->planes_changed = false; -@@ -213,6 +217,9 @@ void __drm_atomic_helper_crtc_destroy_state(struct drm_crtc_state *state) - drm_property_blob_put(state->degamma_lut); - drm_property_blob_put(state->ctm); - drm_property_blob_put(state->gamma_lut); -+#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) -+ drm_property_blob_put(state->cubic_lut); -+#endif - } - EXPORT_SYMBOL(__drm_atomic_helper_crtc_destroy_state); - -diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c -index 25c269bc4..975ece7e0 100644 ---- a/drivers/gpu/drm/drm_atomic_uapi.c -+++ b/drivers/gpu/drm/drm_atomic_uapi.c -@@ -459,6 +459,16 @@ static int drm_atomic_crtc_set_property(struct drm_crtc *crtc, - &replaced); - state->color_mgmt_changed |= replaced; - return ret; -+#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) -+ } else if (property == config->cubic_lut_property) { -+ ret = drm_atomic_replace_property_blob_from_id(dev, -+ &state->cubic_lut, -+ val, -+ -1, sizeof(struct drm_color_lut), -+ &replaced); -+ state->color_mgmt_changed |= replaced; -+ return ret; -+#endif - } else if (property == config->prop_out_fence_ptr) { - s32 __user *fence_ptr = u64_to_user_ptr(val); - -@@ -501,6 +511,10 @@ drm_atomic_crtc_get_property(struct drm_crtc *crtc, - *val = (state->ctm) ? state->ctm->base.id : 0; - else if (property == config->gamma_lut_property) - *val = (state->gamma_lut) ? state->gamma_lut->base.id : 0; -+#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) -+ else if (property == config->cubic_lut_property) -+ *val = (state->cubic_lut) ? state->cubic_lut->base.id : 0; -+#endif - else if (property == config->prop_out_fence_ptr) - *val = 0; - else if (crtc->funcs->atomic_get_property) -diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c -index c7adbeaf1..232abbba3 100644 ---- a/drivers/gpu/drm/drm_auth.c -+++ b/drivers/gpu/drm/drm_auth.c -@@ -135,18 +135,16 @@ static void drm_set_master(struct drm_device *dev, struct drm_file *fpriv, - static int drm_new_set_master(struct drm_device *dev, struct drm_file *fpriv) - { - struct drm_master *old_master; -- struct drm_master *new_master; - - lockdep_assert_held_once(&dev->master_mutex); - - WARN_ON(fpriv->is_master); - old_master = fpriv->master; -- new_master = drm_master_create(dev); -- if (!new_master) -+ fpriv->master = drm_master_create(dev); -+ if (!fpriv->master) { -+ fpriv->master = old_master; - return -ENOMEM; -- spin_lock(&fpriv->master_lookup_lock); -- fpriv->master = new_master; -- spin_unlock(&fpriv->master_lookup_lock); -+ } - - fpriv->is_master = 1; - fpriv->authenticated = 1; -@@ -304,13 +302,10 @@ int drm_master_open(struct drm_file *file_priv) - /* if there is no current master make this fd it, but do not create - * any master object for render clients */ - mutex_lock(&dev->master_mutex); -- if (!dev->master) { -+ if (!dev->master) - ret = drm_new_set_master(dev, file_priv); -- } else { -- spin_lock(&file_priv->master_lookup_lock); -+ else - file_priv->master = drm_master_get(dev->master); -- spin_unlock(&file_priv->master_lookup_lock); -- } - mutex_unlock(&dev->master_mutex); - - return ret; -@@ -376,31 +371,6 @@ struct drm_master *drm_master_get(struct drm_master *master) - } - EXPORT_SYMBOL(drm_master_get); - --/** -- * drm_file_get_master - reference &drm_file.master of @file_priv -- * @file_priv: DRM file private -- * -- * Increments the reference count of @file_priv's &drm_file.master and returns -- * the &drm_file.master. If @file_priv has no &drm_file.master, returns NULL. -- * -- * Master pointers returned from this function should be unreferenced using -- * drm_master_put(). -- */ --struct drm_master *drm_file_get_master(struct drm_file *file_priv) --{ -- struct drm_master *master = NULL; -- -- spin_lock(&file_priv->master_lookup_lock); -- if (!file_priv->master) -- goto unlock; -- master = drm_master_get(file_priv->master); -- --unlock: -- spin_unlock(&file_priv->master_lookup_lock); -- return master; --} --EXPORT_SYMBOL(drm_file_get_master); -- - static void drm_master_destroy(struct kref *kref) - { - struct drm_master *master = container_of(kref, struct drm_master, refcount); -diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c -index 138ff34b3..7b270b68a 100644 ---- a/drivers/gpu/drm/drm_color_mgmt.c -+++ b/drivers/gpu/drm/drm_color_mgmt.c -@@ -33,7 +33,7 @@ - /** - * DOC: overview - * -- * Color management or color space adjustments is supported through a set of 5 -+ * Color management or color space adjustments is supported through a set of 7 - * properties on the &drm_crtc object. They are set up by calling - * drm_crtc_enable_color_mgmt(). - * -@@ -60,7 +60,7 @@ - * “CTMâ€: - * Blob property to set the current transformation matrix (CTM) apply to - * pixel data after the lookup through the degamma LUT and before the -- * lookup through the gamma LUT. The data is interpreted as a struct -+ * lookup through the cubic LUT. The data is interpreted as a struct - * &drm_color_ctm. - * - * Setting this to NULL (blob property value set to 0) means a -@@ -68,13 +68,40 @@ - * boot-up state too. Drivers can access the blob for the color conversion - * matrix through &drm_crtc_state.ctm. - * -+ * â€CUBIC_LUTâ€: -+ * Blob property to set the cubic (3D) lookup table performing color -+ * mapping after the transformation matrix and before the lookup through -+ * the gamma LUT. Unlike the degamma and gamma LUTs that map color -+ * components independently, the 3D LUT converts an input color to an -+ * output color by indexing into the 3D table using the color components -+ * as a 3D coordinate. The LUT is subsampled as 8-bit (or more) precision -+ * would require too much storage space in the hardware, so the precision -+ * of the color components is reduced before the look up, and the low -+ * order bits may be used to interpolate between the nearest points in 3D -+ * space. -+ * -+ * The data is interpreted as an array of &struct drm_color_lut elements. -+ * Hardware might choose not to use the full precision of the LUT -+ * elements. -+ * -+ * Setting this to NULL (blob property value set to 0) means the output -+ * color is identical to the input color. This is generally the driver -+ * boot-up state too. Drivers can access this blob through -+ * &drm_crtc_state.cubic_lut. -+ * -+ * â€CUBIC_LUT_SIZEâ€: -+ * Unsigned range property to give the size of the lookup table to be set -+ * on the CUBIC_LUT property (the size depends on the underlying hardware). -+ * If drivers support multiple LUT sizes then they should publish the -+ * largest size, and sub-sample smaller sized LUTs appropriately. -+ * - * “GAMMA_LUTâ€: - * Blob property to set the gamma lookup table (LUT) mapping pixel data -- * after the transformation matrix to data sent to the connector. The -- * data is interpreted as an array of &struct drm_color_lut elements. -- * Hardware might choose not to use the full precision of the LUT elements -- * nor use all the elements of the LUT (for example the hardware might -- * choose to interpolate between LUT[0] and LUT[4]). -+ * after the cubic LUT to data sent to the connector. The data is -+ * interpreted as an array of &struct drm_color_lut elements. Hardware -+ * might choose not to use the full precision of the LUT elements nor use -+ * all the elements of the LUT (for example the hardware might choose to -+ * interpolate between LUT[0] and LUT[4]). - * - * Setting this to NULL (blob property value set to 0) means a - * linear/pass-thru gamma table should be used. This is generally the -diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c -index b0a826489..3d7182001 100644 ---- a/drivers/gpu/drm/drm_debugfs.c -+++ b/drivers/gpu/drm/drm_debugfs.c -@@ -91,7 +91,6 @@ static int drm_clients_info(struct seq_file *m, void *data) - mutex_lock(&dev->filelist_mutex); - list_for_each_entry_reverse(priv, &dev->filelist, lhead) { - struct task_struct *task; -- bool is_current_master = drm_is_current_master(priv); - - rcu_read_lock(); /* locks pid_task()->comm */ - task = pid_task(priv->pid, PIDTYPE_PID); -@@ -100,7 +99,7 @@ static int drm_clients_info(struct seq_file *m, void *data) - task ? task->comm : "", - pid_vnr(priv->pid), - priv->minor->index, -- is_current_master ? 'y' : 'n', -+ drm_is_current_master(priv) ? 'y' : 'n', - priv->authenticated ? 'y' : 'n', - from_kuid_munged(seq_user_ns(m), uid), - priv->magic); -diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c -index add317bd8..cfb96e181 100644 ---- a/drivers/gpu/drm/drm_edid.c -+++ b/drivers/gpu/drm/drm_edid.c -@@ -1835,20 +1835,11 @@ static void connector_bad_edid(struct drm_connector *connector, - u8 *edid, int num_blocks) - { - int i; -- u8 last_block; -- -- /* -- * 0x7e in the EDID is the number of extension blocks. The EDID -- * is 1 (base block) + num_ext_blocks big. That means we can think -- * of 0x7e in the EDID of the _index_ of the last block in the -- * combined chunk of memory. -- */ -- last_block = edid[0x7e]; -+ u8 num_of_ext = edid[0x7e]; - - /* Calculate real checksum for the last edid extension block data */ -- if (last_block < num_blocks) -- connector->real_edid_checksum = -- drm_edid_block_checksum(edid + last_block * EDID_LENGTH); -+ connector->real_edid_checksum = -+ drm_edid_block_checksum(edid + num_of_ext * EDID_LENGTH); - - if (connector->bad_edid_counter++ && !drm_debug_enabled(DRM_UT_KMS)) - return; -@@ -4860,6 +4851,43 @@ static void drm_parse_vcdb(struct drm_connector *connector, const u8 *db) - info->rgb_quant_range_selectable = true; - } - -+#ifdef CONFIG_NO_GKI -+static -+void drm_get_max_frl_rate(int max_frl_rate, u8 *max_lanes, u8 *max_rate_per_lane) -+{ -+ switch (max_frl_rate) { -+ case 1: -+ *max_lanes = 3; -+ *max_rate_per_lane = 3; -+ break; -+ case 2: -+ *max_lanes = 3; -+ *max_rate_per_lane = 6; -+ break; -+ case 3: -+ *max_lanes = 4; -+ *max_rate_per_lane = 6; -+ break; -+ case 4: -+ *max_lanes = 4; -+ *max_rate_per_lane = 8; -+ break; -+ case 5: -+ *max_lanes = 4; -+ *max_rate_per_lane = 10; -+ break; -+ case 6: -+ *max_lanes = 4; -+ *max_rate_per_lane = 12; -+ break; -+ case 0: -+ default: -+ *max_lanes = 0; -+ *max_rate_per_lane = 0; -+ } -+} -+#endif -+ - static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector, - const u8 *db) - { -@@ -4913,6 +4941,76 @@ static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector, - } - } - -+#ifdef CONFIG_NO_GKI -+ if (hf_vsdb[7]) { -+ u8 max_frl_rate; -+ u8 dsc_max_frl_rate; -+ u8 dsc_max_slices; -+ struct drm_hdmi_dsc_cap *hdmi_dsc = &hdmi->dsc_cap; -+ -+ DRM_DEBUG_KMS("hdmi_21 sink detected. parsing edid\n"); -+ max_frl_rate = (hf_vsdb[7] & DRM_EDID_MAX_FRL_RATE_MASK) >> 4; -+ drm_get_max_frl_rate(max_frl_rate, &hdmi->max_lanes, -+ &hdmi->max_frl_rate_per_lane); -+ hdmi_dsc->v_1p2 = hf_vsdb[11] & DRM_EDID_DSC_1P2; -+ -+ if (hdmi_dsc->v_1p2) { -+ hdmi_dsc->native_420 = hf_vsdb[11] & DRM_EDID_DSC_NATIVE_420; -+ hdmi_dsc->all_bpp = hf_vsdb[11] & DRM_EDID_DSC_ALL_BPP; -+ -+ if (hf_vsdb[11] & DRM_EDID_DSC_16BPC) -+ hdmi_dsc->bpc_supported = 16; -+ else if (hf_vsdb[11] & DRM_EDID_DSC_12BPC) -+ hdmi_dsc->bpc_supported = 12; -+ else if (hf_vsdb[11] & DRM_EDID_DSC_10BPC) -+ hdmi_dsc->bpc_supported = 10; -+ else -+ hdmi_dsc->bpc_supported = 0; -+ -+ dsc_max_frl_rate = (hf_vsdb[12] & DRM_EDID_DSC_MAX_FRL_RATE_MASK) >> 4; -+ drm_get_max_frl_rate(dsc_max_frl_rate, &hdmi_dsc->max_lanes, -+ &hdmi_dsc->max_frl_rate_per_lane); -+ hdmi_dsc->total_chunk_kbytes = hf_vsdb[13] & DRM_EDID_DSC_TOTAL_CHUNK_KBYTES; -+ -+ dsc_max_slices = hf_vsdb[12] & DRM_EDID_DSC_MAX_SLICES; -+ switch (dsc_max_slices) { -+ case 1: -+ hdmi_dsc->max_slices = 1; -+ hdmi_dsc->clk_per_slice = 340; -+ break; -+ case 2: -+ hdmi_dsc->max_slices = 2; -+ hdmi_dsc->clk_per_slice = 340; -+ break; -+ case 3: -+ hdmi_dsc->max_slices = 4; -+ hdmi_dsc->clk_per_slice = 340; -+ break; -+ case 4: -+ hdmi_dsc->max_slices = 8; -+ hdmi_dsc->clk_per_slice = 340; -+ break; -+ case 5: -+ hdmi_dsc->max_slices = 8; -+ hdmi_dsc->clk_per_slice = 400; -+ break; -+ case 6: -+ hdmi_dsc->max_slices = 12; -+ hdmi_dsc->clk_per_slice = 400; -+ break; -+ case 7: -+ hdmi_dsc->max_slices = 16; -+ hdmi_dsc->clk_per_slice = 400; -+ break; -+ case 0: -+ default: -+ hdmi_dsc->max_slices = 0; -+ hdmi_dsc->clk_per_slice = 0; -+ } -+ } -+ } -+#endif -+ - drm_parse_ycbcr420_deep_color_info(connector, hf_vsdb); - } - -diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c -index 537e7de8e..01670305d 100644 ---- a/drivers/gpu/drm/drm_file.c -+++ b/drivers/gpu/drm/drm_file.c -@@ -177,7 +177,6 @@ struct drm_file *drm_file_alloc(struct drm_minor *minor) - init_waitqueue_head(&file->event_wait); - file->event_space = 4096; /* set aside 4k for event buffer */ - -- spin_lock_init(&file->master_lookup_lock); - mutex_init(&file->event_read_lock); - - if (drm_core_check_feature(dev, DRIVER_GEM)) -@@ -776,20 +775,19 @@ void drm_event_cancel_free(struct drm_device *dev, - EXPORT_SYMBOL(drm_event_cancel_free); - - /** -- * drm_send_event_locked - send DRM event to file descriptor -+ * drm_send_event_helper - send DRM event to file descriptor - * @dev: DRM device - * @e: DRM event to deliver -+ * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC -+ * time domain - * -- * This function sends the event @e, initialized with drm_event_reserve_init(), -- * to its associated userspace DRM file. Callers must already hold -- * &drm_device.event_lock, see drm_send_event() for the unlocked version. -- * -- * Note that the core will take care of unlinking and disarming events when the -- * corresponding DRM file is closed. Drivers need not worry about whether the -- * DRM file for this event still exists and can call this function upon -- * completion of the asynchronous work unconditionally. -+ * This helper function sends the event @e, initialized with -+ * drm_event_reserve_init(), to its associated userspace DRM file. -+ * The timestamp variant of dma_fence_signal is used when the caller -+ * sends a valid timestamp. - */ --void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) -+void drm_send_event_helper(struct drm_device *dev, -+ struct drm_pending_event *e, ktime_t timestamp) - { - assert_spin_locked(&dev->event_lock); - -@@ -800,7 +798,10 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) - } - - if (e->fence) { -- dma_fence_signal(e->fence); -+ if (timestamp) -+ dma_fence_signal_timestamp(e->fence, timestamp); -+ else -+ dma_fence_signal(e->fence); - dma_fence_put(e->fence); - } - -@@ -815,6 +816,48 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) - wake_up_interruptible_poll(&e->file_priv->event_wait, - EPOLLIN | EPOLLRDNORM); - } -+ -+/** -+ * drm_send_event_timestamp_locked - send DRM event to file descriptor -+ * @dev: DRM device -+ * @e: DRM event to deliver -+ * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC -+ * time domain -+ * -+ * This function sends the event @e, initialized with drm_event_reserve_init(), -+ * to its associated userspace DRM file. Callers must already hold -+ * &drm_device.event_lock. -+ * -+ * Note that the core will take care of unlinking and disarming events when the -+ * corresponding DRM file is closed. Drivers need not worry about whether the -+ * DRM file for this event still exists and can call this function upon -+ * completion of the asynchronous work unconditionally. -+ */ -+void drm_send_event_timestamp_locked(struct drm_device *dev, -+ struct drm_pending_event *e, ktime_t timestamp) -+{ -+ drm_send_event_helper(dev, e, timestamp); -+} -+EXPORT_SYMBOL(drm_send_event_timestamp_locked); -+ -+/** -+ * drm_send_event_locked - send DRM event to file descriptor -+ * @dev: DRM device -+ * @e: DRM event to deliver -+ * -+ * This function sends the event @e, initialized with drm_event_reserve_init(), -+ * to its associated userspace DRM file. Callers must already hold -+ * &drm_device.event_lock, see drm_send_event() for the unlocked version. -+ * -+ * Note that the core will take care of unlinking and disarming events when the -+ * corresponding DRM file is closed. Drivers need not worry about whether the -+ * DRM file for this event still exists and can call this function upon -+ * completion of the asynchronous work unconditionally. -+ */ -+void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) -+{ -+ drm_send_event_helper(dev, e, 0); -+} - EXPORT_SYMBOL(drm_send_event_locked); - - /** -@@ -837,7 +880,7 @@ void drm_send_event(struct drm_device *dev, struct drm_pending_event *e) - unsigned long irqflags; - - spin_lock_irqsave(&dev->event_lock, irqflags); -- drm_send_event_locked(dev, e); -+ drm_send_event_helper(dev, e, 0); - spin_unlock_irqrestore(&dev->event_lock, irqflags); - } - EXPORT_SYMBOL(drm_send_event); -diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c -index 722c7ebe4..e2241a5fd 100644 ---- a/drivers/gpu/drm/drm_fourcc.c -+++ b/drivers/gpu/drm/drm_fourcc.c -@@ -278,6 +278,16 @@ const struct drm_format_info *__drm_format_info(u32 format) - .num_planes = 2, .char_per_block = { 5, 5, 0 }, - .block_w = { 4, 2, 0 }, .block_h = { 1, 1, 0 }, .hsub = 2, - .vsub = 2, .is_yuv = true }, -+#ifdef CONFIG_NO_GKI -+ { .format = DRM_FORMAT_NV20, .depth = 0, -+ .num_planes = 2, .char_per_block = { 5, 5, 0 }, -+ .block_w = { 4, 2, 0 }, .block_h = { 1, 1, 0 }, .hsub = 2, -+ .vsub = 1, .is_yuv = true }, -+ { .format = DRM_FORMAT_NV30, .depth = 0, -+ .num_planes = 2, .char_per_block = { 5, 5, 0 }, -+ .block_w = { 4, 2, 0 }, .block_h = { 1, 1, 0 }, .hsub = 1, -+ .vsub = 1, .is_yuv = true }, -+#endif - { .format = DRM_FORMAT_Q410, .depth = 0, - .num_planes = 3, .char_per_block = { 2, 2, 2 }, - .block_w = { 1, 1, 1 }, .block_h = { 1, 1, 1 }, .hsub = 0, -diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c -index 4606cc938..83fc51b13 100644 ---- a/drivers/gpu/drm/drm_ioctl.c -+++ b/drivers/gpu/drm/drm_ioctl.c -@@ -537,6 +537,7 @@ int drm_version(struct drm_device *dev, void *data, - */ - int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) - { -+#ifndef CONFIG_DRM_IGNORE_IOTCL_PERMIT - /* ROOT_ONLY is only for CAP_SYS_ADMIN */ - if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN))) - return -EACCES; -@@ -555,6 +556,7 @@ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) - if (unlikely(!(flags & DRM_RENDER_ALLOW) && - drm_is_render_client(file_priv))) - return -EACCES; -+#endif - - return 0; - } -@@ -678,9 +680,9 @@ static const struct drm_ioctl_desc drm_ioctls[] = { - DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb_ioctl, 0), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER), -- DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, 0), -- DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, 0), -- DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, 0), -+ DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_RENDER_ALLOW), -+ DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_RENDER_ALLOW), -+ DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, 0), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER), - DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR2, drm_mode_cursor2_ioctl, DRM_MASTER), -diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c -index aef226340..da4f085fc 100644 ---- a/drivers/gpu/drm/drm_lease.c -+++ b/drivers/gpu/drm/drm_lease.c -@@ -107,19 +107,10 @@ static bool _drm_has_leased(struct drm_master *master, int id) - */ - bool _drm_lease_held(struct drm_file *file_priv, int id) - { -- bool ret; -- struct drm_master *master; -- -- if (!file_priv) -+ if (!file_priv || !file_priv->master) - return true; - -- master = drm_file_get_master(file_priv); -- if (!master) -- return true; -- ret = _drm_lease_held_master(master, id); -- drm_master_put(&master); -- -- return ret; -+ return _drm_lease_held_master(file_priv->master, id); - } - - /** -@@ -138,22 +129,13 @@ bool drm_lease_held(struct drm_file *file_priv, int id) - struct drm_master *master; - bool ret; - -- if (!file_priv) -+ if (!file_priv || !file_priv->master || !file_priv->master->lessor) - return true; - -- master = drm_file_get_master(file_priv); -- if (!master) -- return true; -- if (!master->lessor) { -- ret = true; -- goto out; -- } -+ master = file_priv->master; - mutex_lock(&master->dev->mode_config.idr_mutex); - ret = _drm_lease_held_master(master, id); - mutex_unlock(&master->dev->mode_config.idr_mutex); -- --out: -- drm_master_put(&master); - return ret; - } - -@@ -173,16 +155,10 @@ uint32_t drm_lease_filter_crtcs(struct drm_file *file_priv, uint32_t crtcs_in) - int count_in, count_out; - uint32_t crtcs_out = 0; - -- if (!file_priv) -+ if (!file_priv || !file_priv->master || !file_priv->master->lessor) - return crtcs_in; - -- master = drm_file_get_master(file_priv); -- if (!master) -- return crtcs_in; -- if (!master->lessor) { -- crtcs_out = crtcs_in; -- goto out; -- } -+ master = file_priv->master; - dev = master->dev; - - count_in = count_out = 0; -@@ -201,9 +177,6 @@ uint32_t drm_lease_filter_crtcs(struct drm_file *file_priv, uint32_t crtcs_in) - count_in++; - } - mutex_unlock(&master->dev->mode_config.idr_mutex); -- --out: -- drm_master_put(&master); - return crtcs_out; - } - -@@ -517,7 +490,7 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, - size_t object_count; - int ret = 0; - struct idr leases; -- struct drm_master *lessor; -+ struct drm_master *lessor = lessor_priv->master; - struct drm_master *lessee = NULL; - struct file *lessee_file = NULL; - struct file *lessor_file = lessor_priv->filp; -@@ -529,6 +502,12 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - return -EOPNOTSUPP; - -+ /* Do not allow sub-leases */ -+ if (lessor->lessor) { -+ DRM_DEBUG_LEASE("recursive leasing not allowed\n"); -+ return -EINVAL; -+ } -+ - /* need some objects */ - if (cl->object_count == 0) { - DRM_DEBUG_LEASE("no objects in lease\n"); -@@ -540,22 +519,12 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, - return -EINVAL; - } - -- lessor = drm_file_get_master(lessor_priv); -- /* Do not allow sub-leases */ -- if (lessor->lessor) { -- DRM_DEBUG_LEASE("recursive leasing not allowed\n"); -- ret = -EINVAL; -- goto out_lessor; -- } -- - object_count = cl->object_count; - - object_ids = memdup_user(u64_to_user_ptr(cl->object_ids), - array_size(object_count, sizeof(__u32))); -- if (IS_ERR(object_ids)) { -- ret = PTR_ERR(object_ids); -- goto out_lessor; -- } -+ if (IS_ERR(object_ids)) -+ return PTR_ERR(object_ids); - - idr_init(&leases); - -@@ -566,15 +535,14 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, - if (ret) { - DRM_DEBUG_LEASE("lease object lookup failed: %i\n", ret); - idr_destroy(&leases); -- goto out_lessor; -+ return ret; - } - - /* Allocate a file descriptor for the lease */ - fd = get_unused_fd_flags(cl->flags & (O_CLOEXEC | O_NONBLOCK)); - if (fd < 0) { - idr_destroy(&leases); -- ret = fd; -- goto out_lessor; -+ return fd; - } - - DRM_DEBUG_LEASE("Creating lease\n"); -@@ -610,7 +578,6 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, - /* Hook up the fd */ - fd_install(fd, lessee_file); - -- drm_master_put(&lessor); - DRM_DEBUG_LEASE("drm_mode_create_lease_ioctl succeeded\n"); - return 0; - -@@ -620,8 +587,6 @@ int drm_mode_create_lease_ioctl(struct drm_device *dev, - out_leases: - put_unused_fd(fd); - --out_lessor: -- drm_master_put(&lessor); - DRM_DEBUG_LEASE("drm_mode_create_lease_ioctl failed: %d\n", ret); - return ret; - } -@@ -644,7 +609,7 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev, - struct drm_mode_list_lessees *arg = data; - __u32 __user *lessee_ids = (__u32 __user *) (uintptr_t) (arg->lessees_ptr); - __u32 count_lessees = arg->count_lessees; -- struct drm_master *lessor, *lessee; -+ struct drm_master *lessor = lessor_priv->master, *lessee; - int count; - int ret = 0; - -@@ -655,7 +620,6 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev, - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - return -EOPNOTSUPP; - -- lessor = drm_file_get_master(lessor_priv); - DRM_DEBUG_LEASE("List lessees for %d\n", lessor->lessee_id); - - mutex_lock(&dev->mode_config.idr_mutex); -@@ -679,7 +643,6 @@ int drm_mode_list_lessees_ioctl(struct drm_device *dev, - arg->count_lessees = count; - - mutex_unlock(&dev->mode_config.idr_mutex); -- drm_master_put(&lessor); - - return ret; - } -@@ -699,7 +662,7 @@ int drm_mode_get_lease_ioctl(struct drm_device *dev, - struct drm_mode_get_lease *arg = data; - __u32 __user *object_ids = (__u32 __user *) (uintptr_t) (arg->objects_ptr); - __u32 count_objects = arg->count_objects; -- struct drm_master *lessee; -+ struct drm_master *lessee = lessee_priv->master; - struct idr *object_idr; - int count; - void *entry; -@@ -713,7 +676,6 @@ int drm_mode_get_lease_ioctl(struct drm_device *dev, - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - return -EOPNOTSUPP; - -- lessee = drm_file_get_master(lessee_priv); - DRM_DEBUG_LEASE("get lease for %d\n", lessee->lessee_id); - - mutex_lock(&dev->mode_config.idr_mutex); -@@ -741,7 +703,6 @@ int drm_mode_get_lease_ioctl(struct drm_device *dev, - arg->count_objects = count; - - mutex_unlock(&dev->mode_config.idr_mutex); -- drm_master_put(&lessee); - - return ret; - } -@@ -760,7 +721,7 @@ int drm_mode_revoke_lease_ioctl(struct drm_device *dev, - void *data, struct drm_file *lessor_priv) - { - struct drm_mode_revoke_lease *arg = data; -- struct drm_master *lessor; -+ struct drm_master *lessor = lessor_priv->master; - struct drm_master *lessee; - int ret = 0; - -@@ -770,7 +731,6 @@ int drm_mode_revoke_lease_ioctl(struct drm_device *dev, - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - return -EOPNOTSUPP; - -- lessor = drm_file_get_master(lessor_priv); - mutex_lock(&dev->mode_config.idr_mutex); - - lessee = _drm_find_lessee(lessor, arg->lessee_id); -@@ -791,7 +751,6 @@ int drm_mode_revoke_lease_ioctl(struct drm_device *dev, - - fail: - mutex_unlock(&dev->mode_config.idr_mutex); -- drm_master_put(&lessor); - - return ret; - } -diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c -index 5dd475e82..c73aaab95 100644 ---- a/drivers/gpu/drm/drm_mipi_dsi.c -+++ b/drivers/gpu/drm/drm_mipi_dsi.c -@@ -355,6 +355,7 @@ static ssize_t mipi_dsi_device_transfer(struct mipi_dsi_device *dsi, - - if (dsi->mode_flags & MIPI_DSI_MODE_LPM) - msg->flags |= MIPI_DSI_MSG_USE_LPM; -+ msg->flags |= MIPI_DSI_MSG_LASTCOMMAND; - - return ops->transfer(dsi->host, msg); - } -diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c -index f1affc1bb..f7f21dfc4 100644 ---- a/drivers/gpu/drm/drm_mode_config.c -+++ b/drivers/gpu/drm/drm_mode_config.c -@@ -364,6 +364,22 @@ static int drm_mode_create_standard_properties(struct drm_device *dev) - return -ENOMEM; - dev->mode_config.gamma_lut_size_property = prop; - -+#if defined(CONFIG_ROCKCHIP_DRM_CUBIC_LUT) -+ prop = drm_property_create(dev, -+ DRM_MODE_PROP_BLOB, -+ "CUBIC_LUT", 0); -+ if (!prop) -+ return -ENOMEM; -+ dev->mode_config.cubic_lut_property = prop; -+ -+ prop = drm_property_create_range(dev, -+ DRM_MODE_PROP_IMMUTABLE, -+ "CUBIC_LUT_SIZE", 0, UINT_MAX); -+ if (!prop) -+ return -ENOMEM; -+ dev->mode_config.cubic_lut_size_property = prop; -+#endif -+ - prop = drm_property_create(dev, - DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB, - "IN_FORMATS", 0); -diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c -index 0f99e5453..39b092e33 100644 ---- a/drivers/gpu/drm/drm_modes.c -+++ b/drivers/gpu/drm/drm_modes.c -@@ -1290,8 +1290,7 @@ EXPORT_SYMBOL(drm_mode_prune_invalid); - * Negative if @lh_a is better than @lh_b, zero if they're equivalent, or - * positive if @lh_b is better than @lh_a. - */ --static int drm_mode_compare(void *priv, const struct list_head *lh_a, -- const struct list_head *lh_b) -+static int drm_mode_compare(void *priv, struct list_head *lh_a, struct list_head *lh_b) - { - struct drm_display_mode *a = list_entry(lh_a, struct drm_display_mode, head); - struct drm_display_mode *b = list_entry(lh_b, struct drm_display_mode, head); -@@ -1940,6 +1939,7 @@ void drm_mode_convert_to_umode(struct drm_mode_modeinfo *out, - strncpy(out->name, in->name, DRM_DISPLAY_MODE_LEN); - out->name[DRM_DISPLAY_MODE_LEN-1] = 0; - } -+EXPORT_SYMBOL_GPL(drm_mode_convert_to_umode); - - /** - * drm_crtc_convert_umode - convert a modeinfo into a drm_display_mode -@@ -2016,6 +2016,7 @@ int drm_mode_convert_umode(struct drm_device *dev, - - return 0; - } -+EXPORT_SYMBOL_GPL(drm_mode_convert_umode); - - /** - * drm_mode_is_420_only - if a given videomode can be only supported in YCBCR420 -diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c -index 9f955f201..ca6d13eb3 100644 ---- a/drivers/gpu/drm/drm_prime.c -+++ b/drivers/gpu/drm/drm_prime.c -@@ -780,6 +780,28 @@ int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) - } - EXPORT_SYMBOL(drm_gem_dmabuf_mmap); - -+/** -+ * drm_gem_dmabuf_get_uuid - dma_buf get_uuid implementation for GEM -+ * @dma_buf: buffer to query -+ * @uuid: uuid outparam -+ * -+ * Queries the buffer's virtio UUID. This can be used as the -+ * &dma_buf_ops.get_uuid callback. Calls into &drm_driver.gem_prime_get_uuid. -+ * -+ * Returns 0 on success or a negative error code on failure. -+ */ -+int drm_gem_dmabuf_get_uuid(struct dma_buf *dma_buf, uuid_t *uuid) -+{ -+ struct drm_gem_object *obj = dma_buf->priv; -+ struct drm_device *dev = obj->dev; -+ -+ if (!dev->driver->gem_prime_get_uuid) -+ return -ENODEV; -+ -+ return dev->driver->gem_prime_get_uuid(obj, uuid); -+} -+EXPORT_SYMBOL(drm_gem_dmabuf_get_uuid); -+ - static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { - .cache_sgt_mapping = true, - .attach = drm_gem_map_attach, -@@ -790,6 +812,7 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { - .mmap = drm_gem_dmabuf_mmap, - .vmap = drm_gem_dmabuf_vmap, - .vunmap = drm_gem_dmabuf_vunmap, -+ .get_uuid = drm_gem_dmabuf_get_uuid, - }; - - /** -diff --git a/drivers/gpu/drm/drm_sync_helper.c b/drivers/gpu/drm/drm_sync_helper.c -new file mode 100755 -index 000000000..0c6aa875b ---- /dev/null -+++ b/drivers/gpu/drm/drm_sync_helper.c -@@ -0,0 +1,314 @@ -+/* -+ * drm_sync_helper.c: software fence and helper functions for fences and -+ * reservations used for dma buffer access synchronization between drivers. -+ * -+ * Copyright 2014 Google, Inc. -+ * -+ * This software is licensed under the terms of the GNU General Public -+ * License version 2, as published by the Free Software Foundation, and -+ * may be copied, distributed, and modified under those terms. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ */ -+ -+#include -+#include -+#include -+#include -+ -+static DEFINE_SPINLOCK(sw_fence_lock); -+ -+void drm_add_reservation(struct reservation_object *resv, -+ struct reservation_object **resvs, -+ unsigned long *excl_resvs_bitmap, -+ unsigned int *num_resvs, bool exclusive) -+{ -+ unsigned int r; -+ -+ for (r = 0; r < *num_resvs; r++) { -+ if (resvs[r] == resv) -+ return; -+ } -+ resvs[*num_resvs] = resv; -+ if (exclusive) -+ set_bit(*num_resvs, excl_resvs_bitmap); -+ (*num_resvs)++; -+} -+EXPORT_SYMBOL(drm_add_reservation); -+ -+int drm_lock_reservations(struct reservation_object **resvs, -+ unsigned int num_resvs, struct ww_acquire_ctx *ctx) -+{ -+ unsigned int r; -+ struct reservation_object *slow_res = NULL; -+ -+ ww_acquire_init(ctx, &reservation_ww_class); -+ -+retry: -+ for (r = 0; r < num_resvs; r++) { -+ int ret; -+ /* skip the resv we locked with slow lock */ -+ if (resvs[r] == slow_res) { -+ slow_res = NULL; -+ continue; -+ } -+ ret = ww_mutex_lock(&resvs[r]->lock, ctx); -+ if (ret < 0) { -+ unsigned int slow_r = r; -+ /* -+ * undo all the locks we already done, -+ * in reverse order -+ */ -+ while (r > 0) { -+ r--; -+ ww_mutex_unlock(&resvs[r]->lock); -+ } -+ if (slow_res) -+ ww_mutex_unlock(&slow_res->lock); -+ if (ret == -EDEADLK) { -+ slow_res = resvs[slow_r]; -+ ww_mutex_lock_slow(&slow_res->lock, ctx); -+ goto retry; -+ } -+ ww_acquire_fini(ctx); -+ return ret; -+ } -+ } -+ -+ ww_acquire_done(ctx); -+ return 0; -+} -+EXPORT_SYMBOL(drm_lock_reservations); -+ -+void drm_unlock_reservations(struct reservation_object **resvs, -+ unsigned int num_resvs, -+ struct ww_acquire_ctx *ctx) -+{ -+ unsigned int r; -+ -+ for (r = 0; r < num_resvs; r++) -+ ww_mutex_unlock(&resvs[r]->lock); -+ -+ ww_acquire_fini(ctx); -+} -+EXPORT_SYMBOL(drm_unlock_reservations); -+ -+static void reservation_cb_fence_cb(struct fence *fence, struct fence_cb *cb) -+{ -+ struct drm_reservation_fence_cb *rfcb = -+ container_of(cb, struct drm_reservation_fence_cb, base); -+ struct drm_reservation_cb *rcb = rfcb->parent; -+ -+ if (atomic_dec_and_test(&rcb->count)) -+ schedule_work(&rcb->work); -+} -+ -+static void -+reservation_cb_cleanup(struct drm_reservation_cb *rcb) -+{ -+ unsigned cb; -+ -+ for (cb = 0; cb < rcb->num_fence_cbs; cb++) { -+ if (rcb->fence_cbs[cb]) { -+ fence_remove_callback(rcb->fence_cbs[cb]->fence, -+ &rcb->fence_cbs[cb]->base); -+ fence_put(rcb->fence_cbs[cb]->fence); -+ kfree(rcb->fence_cbs[cb]); -+ rcb->fence_cbs[cb] = NULL; -+ } -+ } -+ kfree(rcb->fence_cbs); -+ rcb->fence_cbs = NULL; -+ rcb->num_fence_cbs = 0; -+} -+ -+static void reservation_cb_work(struct work_struct *pwork) -+{ -+ struct drm_reservation_cb *rcb = -+ container_of(pwork, struct drm_reservation_cb, work); -+ /* -+ * clean up everything before calling the callback, because the callback -+ * may free structure containing rcb and work_struct -+ */ -+ reservation_cb_cleanup(rcb); -+ rcb->func(rcb, rcb->context); -+} -+ -+static int -+reservation_cb_add_fence_cb(struct drm_reservation_cb *rcb, struct fence *fence) -+{ -+ int ret = 0; -+ struct drm_reservation_fence_cb *fence_cb; -+ struct drm_reservation_fence_cb **new_fence_cbs; -+ -+ new_fence_cbs = krealloc(rcb->fence_cbs, -+ (rcb->num_fence_cbs + 1) -+ * sizeof(struct drm_reservation_fence_cb *), -+ GFP_KERNEL); -+ if (!new_fence_cbs) -+ return -ENOMEM; -+ rcb->fence_cbs = new_fence_cbs; -+ -+ fence_cb = kzalloc(sizeof(struct drm_reservation_fence_cb), GFP_KERNEL); -+ if (!fence_cb) -+ return -ENOMEM; -+ -+ /* -+ * do not want for fence to disappear on us while we are waiting for -+ * callback and we need it in case we want to remove callbacks -+ */ -+ fence_get(fence); -+ fence_cb->fence = fence; -+ fence_cb->parent = rcb; -+ rcb->fence_cbs[rcb->num_fence_cbs] = fence_cb; -+ atomic_inc(&rcb->count); -+ ret = fence_add_callback(fence, &fence_cb->base, -+ reservation_cb_fence_cb); -+ if (ret == -ENOENT) { -+ /* already signaled */ -+ atomic_dec(&rcb->count); -+ fence_put(fence_cb->fence); -+ kfree(fence_cb); -+ ret = 0; -+ } else if (ret < 0) { -+ atomic_dec(&rcb->count); -+ fence_put(fence_cb->fence); -+ kfree(fence_cb); -+ return ret; -+ } else { -+ rcb->num_fence_cbs++; -+ } -+ return ret; -+} -+ -+void -+drm_reservation_cb_init(struct drm_reservation_cb *rcb, -+ drm_reservation_cb_func_t func, void *context) -+{ -+ INIT_WORK(&rcb->work, reservation_cb_work); -+ atomic_set(&rcb->count, 1); -+ rcb->num_fence_cbs = 0; -+ rcb->fence_cbs = NULL; -+ rcb->func = func; -+ rcb->context = context; -+} -+EXPORT_SYMBOL(drm_reservation_cb_init); -+ -+int -+drm_reservation_cb_add(struct drm_reservation_cb *rcb, -+ struct reservation_object *resv, bool exclusive) -+{ -+ int ret = 0; -+ struct fence *fence; -+ unsigned shared_count = 0, f; -+ struct fence **shared_fences = NULL; -+ -+ /* enum all the fences in the reservation and add callbacks */ -+ ret = reservation_object_get_fences_rcu(resv, &fence, -+ &shared_count, &shared_fences); -+ if (ret < 0) -+ return ret; -+ -+ if (fence) { -+ ret = reservation_cb_add_fence_cb(rcb, fence); -+ if (ret < 0) { -+ reservation_cb_cleanup(rcb); -+ goto error; -+ } -+ } -+ -+ if (exclusive) { -+ for (f = 0; f < shared_count; f++) { -+ ret = reservation_cb_add_fence_cb(rcb, -+ shared_fences[f]); -+ if (ret < 0) { -+ reservation_cb_cleanup(rcb); -+ goto error; -+ } -+ } -+ } -+ -+error: -+ if (fence) -+ fence_put(fence); -+ if (shared_fences) { -+ for (f = 0; f < shared_count; f++) -+ fence_put(shared_fences[f]); -+ kfree(shared_fences); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(drm_reservation_cb_add); -+ -+void -+drm_reservation_cb_done(struct drm_reservation_cb *rcb) -+{ -+ /* -+ * we need to decrement from initial 1 -+ * and trigger the callback in case all the -+ * fences were already triggered -+ */ -+ if (atomic_dec_and_test(&rcb->count)) { -+ /* -+ * we could call the callback here directly but in case -+ * the callback function needs to lock the same mutex -+ * as our caller it could cause a deadlock, so it is -+ * safer to call it from a worker -+ */ -+ schedule_work(&rcb->work); -+ } -+} -+EXPORT_SYMBOL(drm_reservation_cb_done); -+ -+void -+drm_reservation_cb_fini(struct drm_reservation_cb *rcb) -+{ -+ /* make sure no work will be triggered */ -+ atomic_set(&rcb->count, 0); -+ cancel_work_sync(&rcb->work); -+ reservation_cb_cleanup(rcb); -+} -+EXPORT_SYMBOL(drm_reservation_cb_fini); -+ -+static bool sw_fence_enable_signaling(struct fence *f) -+{ -+ return true; -+} -+ -+static const char *sw_fence_get_get_driver_name(struct fence *fence) -+{ -+ return "drm_sync_helper"; -+} -+ -+static const char *sw_fence_get_timeline_name(struct fence *f) -+{ -+ return "drm_sync.sw"; -+} -+ -+static const struct fence_ops sw_fence_ops = { -+ .get_driver_name = sw_fence_get_get_driver_name, -+ .get_timeline_name = sw_fence_get_timeline_name, -+ .enable_signaling = sw_fence_enable_signaling, -+ .signaled = NULL, -+ .wait = fence_default_wait, -+ .release = NULL -+}; -+ -+struct fence *drm_sw_fence_new(unsigned int context, unsigned seqno) -+{ -+ struct fence *fence; -+ -+ fence = kzalloc(sizeof(*fence), GFP_KERNEL); -+ if (!fence) -+ return ERR_PTR(-ENOMEM); -+ fence_init(fence, -+ &sw_fence_ops, -+ &sw_fence_lock, -+ context, seqno); -+ -+ return fence; -+} -+EXPORT_SYMBOL(drm_sw_fence_new); -diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c -index f135b7959..286edbe1b 100644 ---- a/drivers/gpu/drm/drm_vblank.c -+++ b/drivers/gpu/drm/drm_vblank.c -@@ -1000,7 +1000,14 @@ static void send_vblank_event(struct drm_device *dev, - break; - } - trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); -- drm_send_event_locked(dev, &e->base); -+ /* -+ * Use the same timestamp for any associated fence signal to avoid -+ * mismatch in timestamps for vsync & fence events triggered by the -+ * same HW event. Frameworks like SurfaceFlinger in Android expects the -+ * retire-fence timestamp to match exactly with HW vsync as it uses it -+ * for its software vsync modeling. -+ */ -+ drm_send_event_timestamp_locked(dev, &e->base, now); - } - - /** -diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c -index c940ac3aa..a19537706 100644 ---- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c -+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c -@@ -305,9 +305,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) - engine->i915 = i915; - engine->gt = gt; - engine->uncore = gt->uncore; -+ engine->hw_id = engine->guc_id = info->hw_id; - engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); -- engine->hw_id = info->hw_id; -- engine->guc_id = MAKE_GUC_ID(info->class, info->instance); - - engine->class = info->class; - engine->instance = info->instance; -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c -index 6909da1e1..942c7c187 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c -+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c -@@ -213,6 +213,23 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc) - return flags; - } - -+static u32 guc_ctl_ctxinfo_flags(struct intel_guc *guc) -+{ -+ u32 flags = 0; -+ -+ if (intel_guc_submission_is_used(guc)) { -+ u32 ctxnum, base; -+ -+ base = intel_guc_ggtt_offset(guc, guc->stage_desc_pool); -+ ctxnum = GUC_MAX_STAGE_DESCRIPTORS / 16; -+ -+ base >>= PAGE_SHIFT; -+ flags |= (base << GUC_CTL_BASE_ADDR_SHIFT) | -+ (ctxnum << GUC_CTL_CTXNUM_IN16_SHIFT); -+ } -+ return flags; -+} -+ - static u32 guc_ctl_log_params_flags(struct intel_guc *guc) - { - u32 offset = intel_guc_ggtt_offset(guc, guc->log.vma) >> PAGE_SHIFT; -@@ -274,6 +291,7 @@ static void guc_init_params(struct intel_guc *guc) - - BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); - -+ params[GUC_CTL_CTXINFO] = guc_ctl_ctxinfo_flags(guc); - params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); - params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); - params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c -index 7950d28be..d44061033 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c -+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c -@@ -10,52 +10,11 @@ - - /* - * The Additional Data Struct (ADS) has pointers for different buffers used by -- * the GuC. One single gem object contains the ADS struct itself (guc_ads) and -- * all the extra buffers indirectly linked via the ADS struct's entries. -- * -- * Layout of the ADS blob allocated for the GuC: -- * -- * +---------------------------------------+ <== base -- * | guc_ads | -- * +---------------------------------------+ -- * | guc_policies | -- * +---------------------------------------+ -- * | guc_gt_system_info | -- * +---------------------------------------+ -- * | guc_clients_info | -- * +---------------------------------------+ -- * | guc_ct_pool_entry[size] | -- * +---------------------------------------+ -- * | padding | -- * +---------------------------------------+ <== 4K aligned -- * | private data | -- * +---------------------------------------+ -- * | padding | -- * +---------------------------------------+ <== 4K aligned -+ * the GuC. One single gem object contains the ADS struct itself (guc_ads), the -+ * scheduling policies (guc_policies), a structure describing a collection of -+ * register sets (guc_mmio_reg_state) and some extra pages for the GuC to save -+ * its internal state for sleep. - */ --struct __guc_ads_blob { -- struct guc_ads ads; -- struct guc_policies policies; -- struct guc_gt_system_info system_info; -- struct guc_clients_info clients_info; -- struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; --} __packed; -- --static u32 guc_ads_private_data_size(struct intel_guc *guc) --{ -- return PAGE_ALIGN(guc->fw.private_data_size); --} -- --static u32 guc_ads_private_data_offset(struct intel_guc *guc) --{ -- return PAGE_ALIGN(sizeof(struct __guc_ads_blob)); --} -- --static u32 guc_ads_blob_size(struct intel_guc *guc) --{ -- return guc_ads_private_data_offset(guc) + -- guc_ads_private_data_size(guc); --} - - static void guc_policy_init(struct guc_policy *policy) - { -@@ -89,37 +48,26 @@ static void guc_ct_pool_entries_init(struct guc_ct_pool_entry *pool, u32 num) - memset(pool, 0, num * sizeof(*pool)); - } - --static void guc_mapping_table_init(struct intel_gt *gt, -- struct guc_gt_system_info *system_info) --{ -- unsigned int i, j; -- struct intel_engine_cs *engine; -- enum intel_engine_id id; -- -- /* Table must be set to invalid values for entries not used */ -- for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) -- for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) -- system_info->mapping_table[i][j] = -- GUC_MAX_INSTANCES_PER_CLASS; -- -- for_each_engine(engine, gt, id) { -- u8 guc_class = engine->class; -- -- system_info->mapping_table[guc_class][engine->instance] = -- engine->instance; -- } --} -- - /* - * The first 80 dwords of the register state context, containing the - * execlists and ppgtt registers. - */ - #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) - -+/* The ads obj includes the struct itself and buffers passed to GuC */ -+struct __guc_ads_blob { -+ struct guc_ads ads; -+ struct guc_policies policies; -+ struct guc_mmio_reg_state reg_state; -+ struct guc_gt_system_info system_info; -+ struct guc_clients_info clients_info; -+ struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; -+ u8 reg_state_buffer[GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE]; -+} __packed; -+ - static void __guc_ads_init(struct intel_guc *guc) - { - struct intel_gt *gt = guc_to_gt(guc); -- struct drm_i915_private *i915 = gt->i915; - struct __guc_ads_blob *blob = guc->ads_blob; - const u32 skipped_size = LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE; - u32 base; -@@ -151,25 +99,13 @@ static void __guc_ads_init(struct intel_guc *guc) - } - - /* System info */ -- blob->system_info.engine_enabled_masks[RENDER_CLASS] = 1; -- blob->system_info.engine_enabled_masks[COPY_ENGINE_CLASS] = 1; -- blob->system_info.engine_enabled_masks[VIDEO_DECODE_CLASS] = VDBOX_MASK(gt); -- blob->system_info.engine_enabled_masks[VIDEO_ENHANCEMENT_CLASS] = VEBOX_MASK(gt); -- -- blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = -- hweight8(gt->info.sseu.slice_mask); -- blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = -- gt->info.vdbox_sfc_access; -- -- if (INTEL_GEN(i915) >= 12 && !IS_DGFX(i915)) { -- u32 distdbreg = intel_uncore_read(gt->uncore, -- GEN12_DIST_DBS_POPULATED); -- blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = -- ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & -- GEN12_DOORBELLS_PER_SQIDI) + 1; -- } -+ blob->system_info.slice_enabled = hweight8(gt->info.sseu.slice_mask); -+ blob->system_info.rcs_enabled = 1; -+ blob->system_info.bcs_enabled = 1; - -- guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); -+ blob->system_info.vdbox_enable_mask = VDBOX_MASK(gt); -+ blob->system_info.vebox_enable_mask = VEBOX_MASK(gt); -+ blob->system_info.vdbox_sfc_support_mask = gt->info.vdbox_sfc_access; - - base = intel_guc_ggtt_offset(guc, guc->ads_vma); - -@@ -182,12 +118,11 @@ static void __guc_ads_init(struct intel_guc *guc) - - /* ADS */ - blob->ads.scheduler_policies = base + ptr_offset(blob, policies); -+ blob->ads.reg_state_buffer = base + ptr_offset(blob, reg_state_buffer); -+ blob->ads.reg_state_addr = base + ptr_offset(blob, reg_state); - blob->ads.gt_system_info = base + ptr_offset(blob, system_info); - blob->ads.clients_info = base + ptr_offset(blob, clients_info); - -- /* Private Data */ -- blob->ads.private_data = base + guc_ads_private_data_offset(guc); -- - i915_gem_object_flush_map(guc->ads_vma->obj); - } - -@@ -200,15 +135,14 @@ static void __guc_ads_init(struct intel_guc *guc) - */ - int intel_guc_ads_create(struct intel_guc *guc) - { -- u32 size; -+ const u32 size = PAGE_ALIGN(sizeof(struct __guc_ads_blob)); - int ret; - - GEM_BUG_ON(guc->ads_vma); - -- size = guc_ads_blob_size(guc); -- - ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, - (void **)&guc->ads_blob); -+ - if (ret) - return ret; - -@@ -222,18 +156,6 @@ void intel_guc_ads_destroy(struct intel_guc *guc) - i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); - } - --static void guc_ads_private_data_reset(struct intel_guc *guc) --{ -- u32 size; -- -- size = guc_ads_private_data_size(guc); -- if (!size) -- return; -- -- memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, -- size); --} -- - /** - * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse - * @guc: intel_guc struct -@@ -246,8 +168,5 @@ void intel_guc_ads_reset(struct intel_guc *guc) - { - if (!guc->ads_vma) - return; -- - __guc_ads_init(guc); -- -- guc_ads_private_data_reset(guc); - } -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h -index 79c560d9c..a6b733c14 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h -+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h -@@ -26,8 +26,8 @@ - #define GUC_VIDEO_ENGINE2 4 - #define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) - --#define GUC_MAX_ENGINE_CLASSES 16 --#define GUC_MAX_INSTANCES_PER_CLASS 32 -+#define GUC_MAX_ENGINE_CLASSES 5 -+#define GUC_MAX_INSTANCES_PER_CLASS 16 - - #define GUC_DOORBELL_INVALID 256 - -@@ -62,7 +62,12 @@ - #define GUC_STAGE_DESC_ATTR_PCH BIT(6) - #define GUC_STAGE_DESC_ATTR_TERMINATED BIT(7) - --#define GUC_CTL_LOG_PARAMS 0 -+/* New GuC control data */ -+#define GUC_CTL_CTXINFO 0 -+#define GUC_CTL_CTXNUM_IN16_SHIFT 0 -+#define GUC_CTL_BASE_ADDR_SHIFT 12 -+ -+#define GUC_CTL_LOG_PARAMS 1 - #define GUC_LOG_VALID (1 << 0) - #define GUC_LOG_NOTIFY_ON_HALF_FULL (1 << 1) - #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) -@@ -74,11 +79,11 @@ - #define GUC_LOG_ISR_MASK (0x7 << GUC_LOG_ISR_SHIFT) - #define GUC_LOG_BUF_ADDR_SHIFT 12 - --#define GUC_CTL_WA 1 --#define GUC_CTL_FEATURE 2 -+#define GUC_CTL_WA 2 -+#define GUC_CTL_FEATURE 3 - #define GUC_CTL_DISABLE_SCHEDULER (1 << 14) - --#define GUC_CTL_DEBUG 3 -+#define GUC_CTL_DEBUG 4 - #define GUC_LOG_VERBOSITY_SHIFT 0 - #define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT) - #define GUC_LOG_VERBOSITY_MED (1 << GUC_LOG_VERBOSITY_SHIFT) -@@ -92,37 +97,12 @@ - #define GUC_LOG_DISABLED (1 << 6) - #define GUC_PROFILE_ENABLED (1 << 7) - --#define GUC_CTL_ADS 4 -+#define GUC_CTL_ADS 5 - #define GUC_ADS_ADDR_SHIFT 1 - #define GUC_ADS_ADDR_MASK (0xFFFFF << GUC_ADS_ADDR_SHIFT) - - #define GUC_CTL_MAX_DWORDS (SOFT_SCRATCH_COUNT - 2) /* [1..14] */ - --/* Generic GT SysInfo data types */ --#define GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED 0 --#define GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK 1 --#define GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI 2 --#define GUC_GENERIC_GT_SYSINFO_MAX 16 -- --/* -- * The class goes in bits [0..2] of the GuC ID, the instance in bits [3..6]. -- * Bit 7 can be used for operations that apply to all engine classes&instances. -- */ --#define GUC_ENGINE_CLASS_SHIFT 0 --#define GUC_ENGINE_CLASS_MASK (0x7 << GUC_ENGINE_CLASS_SHIFT) --#define GUC_ENGINE_INSTANCE_SHIFT 3 --#define GUC_ENGINE_INSTANCE_MASK (0xf << GUC_ENGINE_INSTANCE_SHIFT) --#define GUC_ENGINE_ALL_INSTANCES BIT(7) -- --#define MAKE_GUC_ID(class, instance) \ -- (((class) << GUC_ENGINE_CLASS_SHIFT) | \ -- ((instance) << GUC_ENGINE_INSTANCE_SHIFT)) -- --#define GUC_ID_TO_ENGINE_CLASS(guc_id) \ -- (((guc_id) & GUC_ENGINE_CLASS_MASK) >> GUC_ENGINE_CLASS_SHIFT) --#define GUC_ID_TO_ENGINE_INSTANCE(guc_id) \ -- (((guc_id) & GUC_ENGINE_INSTANCE_MASK) >> GUC_ENGINE_INSTANCE_SHIFT) -- - /* Work item for submitting workloads into work queue of GuC. */ - struct guc_wq_item { - u32 header; -@@ -356,6 +336,11 @@ struct guc_policies { - } __packed; - - /* GuC MMIO reg state struct */ -+ -+ -+#define GUC_REGSET_MAX_REGISTERS 64 -+#define GUC_S3_SAVE_SPACE_PAGES 10 -+ - struct guc_mmio_reg { - u32 offset; - u32 value; -@@ -363,18 +348,28 @@ struct guc_mmio_reg { - #define GUC_REGSET_MASKED (1 << 0) - } __packed; - -+struct guc_mmio_regset { -+ struct guc_mmio_reg registers[GUC_REGSET_MAX_REGISTERS]; -+ u32 values_valid; -+ u32 number_of_registers; -+} __packed; -+ - /* GuC register sets */ --struct guc_mmio_reg_set { -- u32 address; -- u16 count; -- u16 reserved; -+struct guc_mmio_reg_state { -+ struct guc_mmio_regset engine_reg[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; -+ u32 reserved[98]; - } __packed; - - /* HW info */ - struct guc_gt_system_info { -- u8 mapping_table[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; -- u32 engine_enabled_masks[GUC_MAX_ENGINE_CLASSES]; -- u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; -+ u32 slice_enabled; -+ u32 rcs_enabled; -+ u32 reserved0; -+ u32 bcs_enabled; -+ u32 vdbox_enable_mask; -+ u32 vdbox_sfc_support_mask; -+ u32 vebox_enable_mask; -+ u32 reserved[9]; - } __packed; - - /* Clients info */ -@@ -395,16 +390,15 @@ struct guc_clients_info { - - /* GuC Additional Data Struct */ - struct guc_ads { -- struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; -- u32 reserved0; -+ u32 reg_state_addr; -+ u32 reg_state_buffer; - u32 scheduler_policies; - u32 gt_system_info; - u32 clients_info; - u32 control_data; - u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; - u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; -- u32 private_data; -- u32 reserved[15]; -+ u32 reserved[16]; - } __packed; - - /* GuC logging structures */ -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h -index b37fc2ffa..1949346e7 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h -+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h -@@ -118,11 +118,6 @@ struct guc_doorbell_info { - #define GEN8_DRB_VALID (1<<0) - #define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) - --#define GEN12_DIST_DBS_POPULATED _MMIO(0xd08) --#define GEN12_DOORBELLS_PER_SQIDI_SHIFT 16 --#define GEN12_DOORBELLS_PER_SQIDI (0xff) --#define GEN12_SQIDIS_DOORBELL_EXIST (0xffff) -- - #define DE_GUCRMR _MMIO(0x44054) - - #define GUC_BCS_RCS_IER _MMIO(0xC550) -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c -index ee4ac3922..80e8b6c3b 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c -+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c -@@ -44,19 +44,23 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, - * List of required GuC and HuC binaries per-platform. - * Must be ordered based on platform + revid, from newer to older. - * -+ * TGL 35.2 is interface-compatible with 33.0 for previous Gens. The deltas -+ * between 33.0 and 35.2 are only related to new additions to support new Gen12 -+ * features. -+ * - * Note that RKL uses the same firmware as TGL. - */ - #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ -- fw_def(ROCKETLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ -- fw_def(TIGERLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ -- fw_def(ELKHARTLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ -- fw_def(ICELAKE, 0, guc_def(icl, 49, 0, 1), huc_def(icl, 9, 0, 0)) \ -- fw_def(COMETLAKE, 5, guc_def(cml, 49, 0, 1), huc_def(cml, 4, 0, 0)) \ -- fw_def(COFFEELAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ -- fw_def(GEMINILAKE, 0, guc_def(glk, 49, 0, 1), huc_def(glk, 4, 0, 0)) \ -- fw_def(KABYLAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ -- fw_def(BROXTON, 0, guc_def(bxt, 49, 0, 1), huc_def(bxt, 2, 0, 0)) \ -- fw_def(SKYLAKE, 0, guc_def(skl, 49, 0, 1), huc_def(skl, 2, 0, 0)) -+ fw_def(ROCKETLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ -+ fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ -+ fw_def(ELKHARTLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ -+ fw_def(ICELAKE, 0, guc_def(icl, 33, 0, 0), huc_def(icl, 9, 0, 0)) \ -+ fw_def(COMETLAKE, 5, guc_def(cml, 33, 0, 0), huc_def(cml, 4, 0, 0)) \ -+ fw_def(COFFEELAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ -+ fw_def(GEMINILAKE, 0, guc_def(glk, 33, 0, 0), huc_def(glk, 4, 0, 0)) \ -+ fw_def(KABYLAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ -+ fw_def(BROXTON, 0, guc_def(bxt, 33, 0, 0), huc_def(bxt, 2, 0, 0)) \ -+ fw_def(SKYLAKE, 0, guc_def(skl, 33, 0, 0), huc_def(skl, 2, 0, 0)) - - #define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \ - "i915/" \ -@@ -367,9 +371,6 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) - } - } - -- if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) -- uc_fw->private_data_size = css->private_data_size; -- - obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h -index 99bb1fe1a..23d3a423a 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h -+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h -@@ -88,8 +88,6 @@ struct intel_uc_fw { - - u32 rsa_size; - u32 ucode_size; -- -- u32 private_data_size; - }; - - #ifdef CONFIG_DRM_I915_DEBUG_GUC -diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h -index e41ffc7a7..029214cde 100644 ---- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h -+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h -@@ -69,11 +69,7 @@ struct uc_css_header { - #define CSS_SW_VERSION_UC_MAJOR (0xFF << 16) - #define CSS_SW_VERSION_UC_MINOR (0xFF << 8) - #define CSS_SW_VERSION_UC_PATCH (0xFF << 0) -- u32 reserved0[13]; -- union { -- u32 private_data_size; /* only applies to GuC */ -- u32 reserved1; -- }; -+ u32 reserved[14]; - u32 header_info; - } __packed; - static_assert(sizeof(struct uc_css_header) == 128); -diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c -index ac96b6ab4..42fc5c813 100644 ---- a/drivers/gpu/drm/nouveau/nouveau_drm.c -+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c -@@ -557,7 +557,6 @@ nouveau_drm_device_init(struct drm_device *dev) - nvkm_dbgopt(nouveau_debug, "DRM"); - - INIT_LIST_HEAD(&drm->clients); -- mutex_init(&drm->clients_lock); - spin_lock_init(&drm->tile.lock); - - /* workaround an odd issue on nvc1 by disabling the device's -@@ -628,7 +627,6 @@ nouveau_drm_device_init(struct drm_device *dev) - static void - nouveau_drm_device_fini(struct drm_device *dev) - { -- struct nouveau_cli *cli, *temp_cli; - struct nouveau_drm *drm = nouveau_drm(dev); - - if (nouveau_pmops_runtime()) { -@@ -653,28 +651,9 @@ nouveau_drm_device_fini(struct drm_device *dev) - nouveau_ttm_fini(drm); - nouveau_vga_fini(drm); - -- /* -- * There may be existing clients from as-yet unclosed files. For now, -- * clean them up here rather than deferring until the file is closed, -- * but this likely not correct if we want to support hot-unplugging -- * properly. -- */ -- mutex_lock(&drm->clients_lock); -- list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) { -- list_del(&cli->head); -- mutex_lock(&cli->mutex); -- if (cli->abi16) -- nouveau_abi16_fini(cli->abi16); -- mutex_unlock(&cli->mutex); -- nouveau_cli_fini(cli); -- kfree(cli); -- } -- mutex_unlock(&drm->clients_lock); -- - nouveau_cli_fini(&drm->client); - nouveau_cli_fini(&drm->master); - nvif_parent_dtor(&drm->parent); -- mutex_destroy(&drm->clients_lock); - kfree(drm); - } - -@@ -813,7 +792,7 @@ nouveau_drm_device_remove(struct drm_device *dev) - struct nvkm_client *client; - struct nvkm_device *device; - -- drm_dev_unplug(dev); -+ drm_dev_unregister(dev); - - dev->irq_enabled = false; - client = nvxx_client(&drm->client.base); -@@ -1107,9 +1086,9 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) - - fpriv->driver_priv = cli; - -- mutex_lock(&drm->clients_lock); -+ mutex_lock(&drm->client.mutex); - list_add(&cli->head, &drm->clients); -- mutex_unlock(&drm->clients_lock); -+ mutex_unlock(&drm->client.mutex); - - done: - if (ret && cli) { -@@ -1127,16 +1106,6 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv) - { - struct nouveau_cli *cli = nouveau_cli(fpriv); - struct nouveau_drm *drm = nouveau_drm(dev); -- int dev_index; -- -- /* -- * The device is gone, and as it currently stands all clients are -- * cleaned up in the removal codepath. In the future this may change -- * so that we can support hot-unplugging, but for now we immediately -- * return to avoid a double-free situation. -- */ -- if (!drm_dev_enter(dev, &dev_index)) -- return; - - pm_runtime_get_sync(dev->dev); - -@@ -1145,15 +1114,14 @@ nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv) - nouveau_abi16_fini(cli->abi16); - mutex_unlock(&cli->mutex); - -- mutex_lock(&drm->clients_lock); -+ mutex_lock(&drm->client.mutex); - list_del(&cli->head); -- mutex_unlock(&drm->clients_lock); -+ mutex_unlock(&drm->client.mutex); - - nouveau_cli_fini(cli); - kfree(cli); - pm_runtime_mark_last_busy(dev->dev); - pm_runtime_put_autosuspend(dev->dev); -- drm_dev_exit(dev_index); - } - - static const struct drm_ioctl_desc -diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h -index 8b252dca0..b8025507a 100644 ---- a/drivers/gpu/drm/nouveau/nouveau_drv.h -+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h -@@ -142,11 +142,6 @@ struct nouveau_drm { - - struct list_head clients; - -- /** -- * @clients_lock: Protects access to the @clients list of &struct nouveau_cli. -- */ -- struct mutex clients_lock; -- - u8 old_pm_cap; - - struct { -diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig -index 6153972e0..b9dbedf8f 100644 ---- a/drivers/gpu/drm/panel/Kconfig -+++ b/drivers/gpu/drm/panel/Kconfig -@@ -233,7 +233,6 @@ config DRM_PANEL_OLIMEX_LCD_OLINUXINO - depends on OF - depends on I2C - depends on BACKLIGHT_CLASS_DEVICE -- select CRC32 - help - The panel is used with different sizes LCDs, from 480x272 to - 1280x800, and 24 bit per pixel. -diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c -index 534dd7414..0145129d7 100644 ---- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c -+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c -@@ -590,14 +590,14 @@ static const struct drm_display_mode k101_im2byl02_default_mode = { - .clock = 69700, - - .hdisplay = 800, -- .hsync_start = 800 + 52, -- .hsync_end = 800 + 52 + 8, -- .htotal = 800 + 52 + 8 + 48, -+ .hsync_start = 800 + 6, -+ .hsync_end = 800 + 6 + 15, -+ .htotal = 800 + 6 + 15 + 16, - - .vdisplay = 1280, -- .vsync_start = 1280 + 16, -- .vsync_end = 1280 + 16 + 6, -- .vtotal = 1280 + 16 + 6 + 15, -+ .vsync_start = 1280 + 8, -+ .vsync_end = 1280 + 8 + 48, -+ .vtotal = 1280 + 8 + 48 + 52, - - .width_mm = 135, - .height_mm = 217, -diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c -index 204674fcc..d8acc4d66 100644 ---- a/drivers/gpu/drm/panel/panel-simple.c -+++ b/drivers/gpu/drm/panel/panel-simple.c -@@ -30,6 +30,7 @@ - #include - - #include